2024-11-12 12:24:10,800 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 12:24:10,817 main DEBUG Took 0.013755 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 12:24:10,817 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 12:24:10,818 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 12:24:10,820 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 12:24:10,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,832 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 12:24:10,849 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,852 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,854 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,854 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,855 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,856 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,857 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,858 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,859 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,859 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,860 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,861 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,861 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,862 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,862 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,863 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,864 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,864 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,865 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 12:24:10,866 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,867 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 12:24:10,869 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 12:24:10,870 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 12:24:10,873 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 12:24:10,874 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 12:24:10,876 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 12:24:10,877 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 12:24:10,887 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 12:24:10,891 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 12:24:10,894 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 12:24:10,895 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 12:24:10,895 main DEBUG createAppenders(={Console}) 2024-11-12 12:24:10,896 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-12 12:24:10,896 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 12:24:10,897 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-12 12:24:10,897 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 12:24:10,898 main DEBUG OutputStream closed 2024-11-12 12:24:10,898 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 12:24:10,899 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 12:24:10,899 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-12 12:24:11,001 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 12:24:11,004 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 12:24:11,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 12:24:11,007 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 12:24:11,008 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 12:24:11,009 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 12:24:11,009 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 12:24:11,010 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 12:24:11,010 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 12:24:11,011 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 12:24:11,011 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 12:24:11,012 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 12:24:11,012 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 12:24:11,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 12:24:11,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 12:24:11,014 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 12:24:11,014 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 12:24:11,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 12:24:11,018 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 12:24:11,019 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-12 12:24:11,019 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 12:24:11,020 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-12T12:24:11,431 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3 2024-11-12 12:24:11,435 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 12:24:11,436 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T12:24:11,446 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-12T12:24:11,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T12:24:11,497 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269, deleteOnExit=true 2024-11-12T12:24:11,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T12:24:11,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/test.cache.data in system properties and HBase conf 2024-11-12T12:24:11,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T12:24:11,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/hadoop.log.dir in system properties and HBase conf 2024-11-12T12:24:11,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T12:24:11,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T12:24:11,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T12:24:11,642 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T12:24:11,772 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T12:24:11,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T12:24:11,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T12:24:11,779 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T12:24:11,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T12:24:11,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T12:24:11,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T12:24:11,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T12:24:11,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T12:24:11,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T12:24:11,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/nfs.dump.dir in system properties and HBase conf 2024-11-12T12:24:11,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/java.io.tmpdir in system properties and HBase conf 2024-11-12T12:24:11,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T12:24:11,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T12:24:11,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T12:24:12,982 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T12:24:13,068 INFO [Time-limited test {}] log.Log(170): Logging initialized @3175ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T12:24:13,144 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T12:24:13,214 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T12:24:13,243 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T12:24:13,244 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T12:24:13,245 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T12:24:13,258 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T12:24:13,262 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/hadoop.log.dir/,AVAILABLE} 2024-11-12T12:24:13,263 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T12:24:13,488 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/java.io.tmpdir/jetty-localhost-44319-hadoop-hdfs-3_4_1-tests_jar-_-any-9008830969838394289/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T12:24:13,495 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:44319} 2024-11-12T12:24:13,496 INFO [Time-limited test {}] server.Server(415): Started @3604ms 2024-11-12T12:24:14,149 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T12:24:14,160 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T12:24:14,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T12:24:14,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T12:24:14,166 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T12:24:14,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/hadoop.log.dir/,AVAILABLE} 2024-11-12T12:24:14,170 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T12:24:14,285 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/java.io.tmpdir/jetty-localhost-39711-hadoop-hdfs-3_4_1-tests_jar-_-any-16366120169883614215/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T12:24:14,286 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39711} 2024-11-12T12:24:14,286 INFO [Time-limited test {}] server.Server(415): Started @4394ms 2024-11-12T12:24:14,351 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T12:24:15,905 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269/data/data2/current/BP-1782012263-172.17.0.3-1731414252404/current, will proceed with Du for space computation calculation, 2024-11-12T12:24:15,905 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269/data/data1/current/BP-1782012263-172.17.0.3-1731414252404/current, will proceed with Du for space computation calculation, 2024-11-12T12:24:15,939 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T12:24:15,998 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb26e31d63a5ea4b with lease ID 0xb59bcd708a0692ae: Processing first storage report for DS-3716a86f-e828-47a2-a25a-f34ffd346008 from datanode DatanodeRegistration(127.0.0.1:33211, datanodeUuid=927edd63-cfe6-496c-9696-870b8024df37, infoPort=45825, infoSecurePort=0, ipcPort=43703, storageInfo=lv=-57;cid=testClusterID;nsid=685958343;c=1731414252404) 2024-11-12T12:24:16,001 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb26e31d63a5ea4b with lease ID 0xb59bcd708a0692ae: from storage DS-3716a86f-e828-47a2-a25a-f34ffd346008 node DatanodeRegistration(127.0.0.1:33211, datanodeUuid=927edd63-cfe6-496c-9696-870b8024df37, infoPort=45825, infoSecurePort=0, ipcPort=43703, storageInfo=lv=-57;cid=testClusterID;nsid=685958343;c=1731414252404), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T12:24:16,002 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb26e31d63a5ea4b with lease ID 0xb59bcd708a0692ae: Processing first storage report for DS-fee4afc7-57e3-437a-9e97-bf71da8e5d8e from datanode DatanodeRegistration(127.0.0.1:33211, datanodeUuid=927edd63-cfe6-496c-9696-870b8024df37, infoPort=45825, infoSecurePort=0, ipcPort=43703, storageInfo=lv=-57;cid=testClusterID;nsid=685958343;c=1731414252404) 2024-11-12T12:24:16,002 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb26e31d63a5ea4b with lease ID 0xb59bcd708a0692ae: from storage DS-fee4afc7-57e3-437a-9e97-bf71da8e5d8e node DatanodeRegistration(127.0.0.1:33211, datanodeUuid=927edd63-cfe6-496c-9696-870b8024df37, infoPort=45825, infoSecurePort=0, ipcPort=43703, storageInfo=lv=-57;cid=testClusterID;nsid=685958343;c=1731414252404), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T12:24:16,049 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3 2024-11-12T12:24:16,123 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269/zookeeper_0, clientPort=59170, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T12:24:16,132 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59170 2024-11-12T12:24:16,146 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T12:24:16,150 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T12:24:16,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741825_1001 (size=7) 2024-11-12T12:24:16,754 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a with version=8 2024-11-12T12:24:16,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/hbase-staging 2024-11-12T12:24:16,842 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T12:24:17,098 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4a91c05c96a4:0 server-side Connection retries=45 2024-11-12T12:24:17,107 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T12:24:17,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T12:24:17,112 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T12:24:17,112 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T12:24:17,112 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T12:24:17,241 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T12:24:17,289 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T12:24:17,297 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T12:24:17,300 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T12:24:17,321 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 34588 (auto-detected) 2024-11-12T12:24:17,322 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-12T12:24:17,339 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41939 2024-11-12T12:24:17,359 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41939 connecting to ZooKeeper ensemble=127.0.0.1:59170 2024-11-12T12:24:17,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:419390x0, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T12:24:17,481 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41939-0x1012f00e5900000 connected 2024-11-12T12:24:17,577 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T12:24:17,579 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T12:24:17,589 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T12:24:17,593 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a, hbase.cluster.distributed=false 2024-11-12T12:24:17,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T12:24:17,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41939 2024-11-12T12:24:17,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41939 2024-11-12T12:24:17,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41939 2024-11-12T12:24:17,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41939 2024-11-12T12:24:17,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41939 2024-11-12T12:24:17,723 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4a91c05c96a4:0 server-side Connection retries=45 2024-11-12T12:24:17,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T12:24:17,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T12:24:17,724 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T12:24:17,725 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T12:24:17,725 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T12:24:17,727 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T12:24:17,729 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T12:24:17,730 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38717 2024-11-12T12:24:17,732 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38717 connecting to ZooKeeper ensemble=127.0.0.1:59170 2024-11-12T12:24:17,734 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T12:24:17,736 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T12:24:17,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387170x0, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T12:24:17,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387170x0, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T12:24:17,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38717-0x1012f00e5900001 connected 2024-11-12T12:24:17,775 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T12:24:17,781 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T12:24:17,784 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T12:24:17,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T12:24:17,790 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38717 2024-11-12T12:24:17,790 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38717 2024-11-12T12:24:17,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38717 2024-11-12T12:24:17,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38717 2024-11-12T12:24:17,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38717 2024-11-12T12:24:17,807 DEBUG [M:0;4a91c05c96a4:41939 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4a91c05c96a4:41939 2024-11-12T12:24:17,808 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:17,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T12:24:17,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T12:24:17,826 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:17,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:17,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T12:24:17,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:17,856 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T12:24:17,857 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4a91c05c96a4,41939,1731414256942 from backup master directory 2024-11-12T12:24:17,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T12:24:17,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:17,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T12:24:17,870 WARN [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T12:24:17,871 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:17,873 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T12:24:17,874 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T12:24:17,923 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/hbase.id] with ID: a16b6acc-0c60-49b2-bc4a-28b799cceb49 2024-11-12T12:24:17,924 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/.tmp/hbase.id 2024-11-12T12:24:17,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741826_1002 (size=42) 2024-11-12T12:24:18,336 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/.tmp/hbase.id]:[hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/hbase.id] 2024-11-12T12:24:18,386 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T12:24:18,391 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T12:24:18,409 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-12T12:24:18,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:18,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:18,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741827_1003 (size=196) 2024-11-12T12:24:18,893 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:24:18,895 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T12:24:18,912 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:18,916 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T12:24:18,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741828_1004 (size=1189) 2024-11-12T12:24:19,368 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store 2024-11-12T12:24:19,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741829_1005 (size=34) 2024-11-12T12:24:19,795 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T12:24:19,798 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:24:19,799 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T12:24:19,800 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T12:24:19,800 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T12:24:19,801 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T12:24:19,801 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T12:24:19,801 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T12:24:19,802 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731414259799Disabling compacts and flushes for region at 1731414259799Disabling writes for close at 1731414259801 (+2 ms)Writing region close event to WAL at 1731414259801Closed at 1731414259801 2024-11-12T12:24:19,804 WARN [master/4a91c05c96a4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/.initializing 2024-11-12T12:24:19,804 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/WALs/4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:19,811 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T12:24:19,825 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4a91c05c96a4%2C41939%2C1731414256942, suffix=, logDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/WALs/4a91c05c96a4,41939,1731414256942, archiveDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/oldWALs, maxLogs=10 2024-11-12T12:24:19,847 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/WALs/4a91c05c96a4,41939,1731414256942/4a91c05c96a4%2C41939%2C1731414256942.1731414259829, exclude list is [], retry=0 2024-11-12T12:24:19,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33211,DS-3716a86f-e828-47a2-a25a-f34ffd346008,DISK] 2024-11-12T12:24:19,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T12:24:19,896 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/WALs/4a91c05c96a4,41939,1731414256942/4a91c05c96a4%2C41939%2C1731414256942.1731414259829 2024-11-12T12:24:19,897 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45825:45825)] 2024-11-12T12:24:19,898 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:24:19,898 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:24:19,901 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,902 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T12:24:19,960 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:19,963 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T12:24:19,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T12:24:19,967 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:19,968 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:19,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T12:24:19,972 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:19,973 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:19,973 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T12:24:19,976 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:19,977 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:19,978 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,981 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,982 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,987 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,988 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:19,992 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T12:24:19,997 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T12:24:20,001 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T12:24:20,003 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69436128, jitterRate=0.03467893600463867}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T12:24:20,009 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731414259913Initializing all the Stores at 1731414259915 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731414259915Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414259916 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414259916Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414259916Cleaning up temporary data from old regions at 1731414259988 (+72 ms)Region opened successfully at 1731414260009 (+21 ms) 2024-11-12T12:24:20,011 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T12:24:20,045 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb15e48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4a91c05c96a4/172.17.0.3:0 2024-11-12T12:24:20,075 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T12:24:20,085 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T12:24:20,085 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T12:24:20,088 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T12:24:20,090 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T12:24:20,095 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T12:24:20,096 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T12:24:20,119 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T12:24:20,127 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T12:24:20,139 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T12:24:20,142 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T12:24:20,143 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T12:24:20,153 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T12:24:20,156 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T12:24:20,161 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T12:24:20,174 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T12:24:20,176 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T12:24:20,185 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T12:24:20,209 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T12:24:20,216 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T12:24:20,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T12:24:20,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T12:24:20,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:20,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:20,230 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4a91c05c96a4,41939,1731414256942, sessionid=0x1012f00e5900000, setting cluster-up flag (Was=false) 2024-11-12T12:24:20,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:20,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:20,301 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T12:24:20,303 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:20,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:20,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:20,364 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T12:24:20,368 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:20,378 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T12:24:20,398 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(746): ClusterId : a16b6acc-0c60-49b2-bc4a-28b799cceb49 2024-11-12T12:24:20,401 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T12:24:20,415 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T12:24:20,415 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T12:24:20,428 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T12:24:20,429 DEBUG [RS:0;4a91c05c96a4:38717 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bbf77f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4a91c05c96a4/172.17.0.3:0 2024-11-12T12:24:20,444 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T12:24:20,445 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4a91c05c96a4:38717 2024-11-12T12:24:20,448 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T12:24:20,448 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T12:24:20,449 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T12:24:20,451 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(2659): reportForDuty to master=4a91c05c96a4,41939,1731414256942 with port=38717, startcode=1731414257691 2024-11-12T12:24:20,453 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T12:24:20,459 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T12:24:20,461 DEBUG [RS:0;4a91c05c96a4:38717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T12:24:20,463 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4a91c05c96a4,41939,1731414256942 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T12:24:20,470 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4a91c05c96a4:0, corePoolSize=5, maxPoolSize=5 2024-11-12T12:24:20,470 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4a91c05c96a4:0, corePoolSize=5, maxPoolSize=5 2024-11-12T12:24:20,470 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4a91c05c96a4:0, corePoolSize=5, maxPoolSize=5 2024-11-12T12:24:20,470 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4a91c05c96a4:0, corePoolSize=5, maxPoolSize=5 2024-11-12T12:24:20,470 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4a91c05c96a4:0, corePoolSize=10, maxPoolSize=10 2024-11-12T12:24:20,470 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,471 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4a91c05c96a4:0, corePoolSize=2, maxPoolSize=2 2024-11-12T12:24:20,471 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,474 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731414290474 2024-11-12T12:24:20,476 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T12:24:20,477 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T12:24:20,477 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T12:24:20,478 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T12:24:20,481 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T12:24:20,482 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T12:24:20,482 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T12:24:20,482 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T12:24:20,485 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:20,486 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T12:24:20,484 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,488 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T12:24:20,489 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T12:24:20,489 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T12:24:20,494 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T12:24:20,495 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36739, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T12:24:20,495 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T12:24:20,497 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4a91c05c96a4:0:becomeActiveMaster-HFileCleaner.large.0-1731414260496,5,FailOnTimeoutGroup] 2024-11-12T12:24:20,497 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4a91c05c96a4:0:becomeActiveMaster-HFileCleaner.small.0-1731414260497,5,FailOnTimeoutGroup] 2024-11-12T12:24:20,498 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,498 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T12:24:20,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741831_1007 (size=1321) 2024-11-12T12:24:20,501 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,501 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41939 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:20,501 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,522 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T12:24:20,523 WARN [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T12:24:20,624 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(2659): reportForDuty to master=4a91c05c96a4,41939,1731414256942 with port=38717, startcode=1731414257691 2024-11-12T12:24:20,627 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41939 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:20,629 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41939 {}] master.ServerManager(517): Registering regionserver=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:20,637 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a 2024-11-12T12:24:20,637 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40685 2024-11-12T12:24:20,637 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T12:24:20,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T12:24:20,723 DEBUG [RS:0;4a91c05c96a4:38717 {}] zookeeper.ZKUtil(111): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:20,724 WARN [RS:0;4a91c05c96a4:38717 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T12:24:20,724 INFO [RS:0;4a91c05c96a4:38717 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T12:24:20,725 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/WALs/4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:20,729 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4a91c05c96a4,38717,1731414257691] 2024-11-12T12:24:20,756 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T12:24:20,771 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T12:24:20,775 INFO [RS:0;4a91c05c96a4:38717 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T12:24:20,775 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,776 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T12:24:20,781 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T12:24:20,782 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,782 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,782 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,783 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,783 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,783 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,783 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4a91c05c96a4:0, corePoolSize=2, maxPoolSize=2 2024-11-12T12:24:20,783 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,783 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,783 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,783 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,784 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,784 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4a91c05c96a4:0, corePoolSize=1, maxPoolSize=1 2024-11-12T12:24:20,784 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4a91c05c96a4:0, corePoolSize=3, maxPoolSize=3 2024-11-12T12:24:20,784 DEBUG [RS:0;4a91c05c96a4:38717 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0, corePoolSize=3, maxPoolSize=3 2024-11-12T12:24:20,787 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,788 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,788 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,788 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,788 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,788 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=4a91c05c96a4,38717,1731414257691-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T12:24:20,805 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T12:24:20,807 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=4a91c05c96a4,38717,1731414257691-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,807 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,807 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.Replication(171): 4a91c05c96a4,38717,1731414257691 started 2024-11-12T12:24:20,823 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:20,823 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1482): Serving as 4a91c05c96a4,38717,1731414257691, RpcServer on 4a91c05c96a4/172.17.0.3:38717, sessionid=0x1012f00e5900001 2024-11-12T12:24:20,824 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T12:24:20,824 DEBUG [RS:0;4a91c05c96a4:38717 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:20,824 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4a91c05c96a4,38717,1731414257691' 2024-11-12T12:24:20,824 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T12:24:20,825 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T12:24:20,826 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T12:24:20,826 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T12:24:20,826 DEBUG [RS:0;4a91c05c96a4:38717 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:20,826 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4a91c05c96a4,38717,1731414257691' 2024-11-12T12:24:20,827 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T12:24:20,827 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T12:24:20,828 DEBUG [RS:0;4a91c05c96a4:38717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T12:24:20,828 INFO [RS:0;4a91c05c96a4:38717 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T12:24:20,828 INFO [RS:0;4a91c05c96a4:38717 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T12:24:20,901 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T12:24:20,901 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a 2024-11-12T12:24:20,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741832_1008 (size=32) 2024-11-12T12:24:20,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:24:20,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T12:24:20,919 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T12:24:20,919 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:20,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T12:24:20,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T12:24:20,923 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T12:24:20,924 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:20,924 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T12:24:20,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T12:24:20,928 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T12:24:20,928 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:20,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T12:24:20,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T12:24:20,933 INFO [RS:0;4a91c05c96a4:38717 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T12:24:20,936 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T12:24:20,936 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:20,937 INFO [RS:0;4a91c05c96a4:38717 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4a91c05c96a4%2C38717%2C1731414257691, suffix=, logDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/WALs/4a91c05c96a4,38717,1731414257691, archiveDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/oldWALs, maxLogs=32 2024-11-12T12:24:20,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T12:24:20,938 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T12:24:20,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740 2024-11-12T12:24:20,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740 2024-11-12T12:24:20,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T12:24:20,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T12:24:20,946 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:24:20,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T12:24:20,961 DEBUG [RS:0;4a91c05c96a4:38717 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/WALs/4a91c05c96a4,38717,1731414257691/4a91c05c96a4%2C38717%2C1731414257691.1731414260940, exclude list is [], retry=0 2024-11-12T12:24:20,962 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T12:24:20,965 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62486416, jitterRate=-0.06887984275817871}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:24:20,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33211,DS-3716a86f-e828-47a2-a25a-f34ffd346008,DISK] 2024-11-12T12:24:20,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731414260913Initializing all the Stores at 1731414260915 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731414260916 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731414260916Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414260916Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731414260916Cleaning up temporary data from old regions at 1731414260945 (+29 ms)Region opened successfully at 1731414260968 (+23 ms) 2024-11-12T12:24:20,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T12:24:20,968 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T12:24:20,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T12:24:20,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T12:24:20,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T12:24:20,970 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T12:24:20,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731414260968Disabling compacts and flushes for region at 1731414260968Disabling writes for close at 1731414260969 (+1 ms)Writing region close event to WAL at 1731414260970 (+1 ms)Closed at 1731414260970 2024-11-12T12:24:20,973 INFO [RS:0;4a91c05c96a4:38717 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/WALs/4a91c05c96a4,38717,1731414257691/4a91c05c96a4%2C38717%2C1731414257691.1731414260940 2024-11-12T12:24:20,974 DEBUG [RS:0;4a91c05c96a4:38717 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45825:45825)] 2024-11-12T12:24:20,974 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T12:24:20,974 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T12:24:20,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T12:24:20,993 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T12:24:20,996 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T12:24:21,148 DEBUG [4a91c05c96a4:41939 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T12:24:21,158 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:21,165 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4a91c05c96a4,38717,1731414257691, state=OPENING 2024-11-12T12:24:21,216 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T12:24:21,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:21,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:24:21,366 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T12:24:21,366 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T12:24:21,368 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T12:24:21,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:24:21,548 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T12:24:21,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45323, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T12:24:21,563 INFO [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T12:24:21,563 INFO [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T12:24:21,564 INFO [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T12:24:21,569 INFO [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4a91c05c96a4%2C38717%2C1731414257691.meta, suffix=.meta, logDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/WALs/4a91c05c96a4,38717,1731414257691, archiveDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/oldWALs, maxLogs=32 2024-11-12T12:24:21,592 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/WALs/4a91c05c96a4,38717,1731414257691/4a91c05c96a4%2C38717%2C1731414257691.meta.1731414261572.meta, exclude list is [], retry=0 2024-11-12T12:24:21,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33211,DS-3716a86f-e828-47a2-a25a-f34ffd346008,DISK] 2024-11-12T12:24:21,600 INFO [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/WALs/4a91c05c96a4,38717,1731414257691/4a91c05c96a4%2C38717%2C1731414257691.meta.1731414261572.meta 2024-11-12T12:24:21,601 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45825:45825)] 2024-11-12T12:24:21,602 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:24:21,604 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T12:24:21,607 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T12:24:21,615 INFO [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T12:24:21,620 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T12:24:21,621 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:24:21,621 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T12:24:21,621 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T12:24:21,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T12:24:21,627 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T12:24:21,627 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:21,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T12:24:21,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T12:24:21,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T12:24:21,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:21,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T12:24:21,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T12:24:21,635 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T12:24:21,635 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:21,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T12:24:21,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T12:24:21,639 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T12:24:21,639 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:21,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T12:24:21,641 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T12:24:21,643 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740 2024-11-12T12:24:21,647 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740 2024-11-12T12:24:21,652 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T12:24:21,652 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T12:24:21,654 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:24:21,658 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T12:24:21,660 INFO [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74839428, jitterRate=0.11519438028335571}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:24:21,660 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T12:24:21,662 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731414261622Writing region info on filesystem at 1731414261622Initializing all the Stores at 1731414261624 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731414261625 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731414261625Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414261625Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731414261625Cleaning up temporary data from old regions at 1731414261652 (+27 ms)Running coprocessor post-open hooks at 1731414261660 (+8 ms)Region opened successfully at 1731414261662 (+2 ms) 2024-11-12T12:24:21,670 INFO [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731414261539 2024-11-12T12:24:21,684 DEBUG [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T12:24:21,684 INFO [RS_OPEN_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T12:24:21,687 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:21,690 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4a91c05c96a4,38717,1731414257691, state=OPEN 2024-11-12T12:24:21,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T12:24:21,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T12:24:21,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T12:24:21,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T12:24:21,858 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:21,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T12:24:21,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4a91c05c96a4,38717,1731414257691 in 488 msec 2024-11-12T12:24:21,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T12:24:21,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 886 msec 2024-11-12T12:24:21,876 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T12:24:21,877 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T12:24:21,902 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:21,904 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:21,932 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:21,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54291, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:21,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5560 sec 2024-11-12T12:24:21,965 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731414261965, completionTime=-1 2024-11-12T12:24:21,969 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T12:24:21,969 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T12:24:22,003 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-12T12:24:22,004 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731414322004 2024-11-12T12:24:22,004 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731414382004 2024-11-12T12:24:22,004 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 34 msec 2024-11-12T12:24:22,007 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a91c05c96a4,41939,1731414256942-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:22,008 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a91c05c96a4,41939,1731414256942-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:22,009 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a91c05c96a4,41939,1731414256942-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:22,011 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4a91c05c96a4:41939, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:22,011 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:22,012 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:22,023 DEBUG [master/4a91c05c96a4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T12:24:22,049 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.178sec 2024-11-12T12:24:22,051 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T12:24:22,053 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T12:24:22,055 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T12:24:22,055 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T12:24:22,055 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T12:24:22,057 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a91c05c96a4,41939,1731414256942-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T12:24:22,057 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a91c05c96a4,41939,1731414256942-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T12:24:22,068 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T12:24:22,069 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T12:24:22,070 INFO [master/4a91c05c96a4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4a91c05c96a4,41939,1731414256942-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T12:24:22,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,115 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T12:24:22,115 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T12:24:22,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:22,122 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:22,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a49a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:22,151 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:22,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,155 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:22,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:22,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:22,217 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=8482 2024-11-12T12:24:22,229 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T12:24:22,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3108930d 2024-11-12T12:24:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T12:24:22,244 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T12:24:22,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T12:24:22,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:24:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:22,275 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T12:24:22,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-12T12:24:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T12:24:22,324 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:22,328 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T12:24:22,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741835_1011 (size=959) 2024-11-12T12:24:22,374 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a 2024-11-12T12:24:22,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741836_1012 (size=53) 2024-11-12T12:24:22,391 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:24:22,392 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 03699d6939471fdc6521cd2a2d843827, disabling compactions & flushes 2024-11-12T12:24:22,392 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:22,392 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:22,392 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. after waiting 0 ms 2024-11-12T12:24:22,392 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:22,392 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:22,392 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 03699d6939471fdc6521cd2a2d843827: Waiting for close lock at 1731414262392Disabling compacts and flushes for region at 1731414262392Disabling writes for close at 1731414262392Writing region close event to WAL at 1731414262392Closed at 1731414262392 2024-11-12T12:24:22,395 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T12:24:22,401 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731414262395"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731414262395"}]},"ts":"1731414262395"} 2024-11-12T12:24:22,407 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T12:24:22,410 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T12:24:22,413 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414262410"}]},"ts":"1731414262410"} 2024-11-12T12:24:22,419 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T12:24:22,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03699d6939471fdc6521cd2a2d843827, ASSIGN}] 2024-11-12T12:24:22,425 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03699d6939471fdc6521cd2a2d843827, ASSIGN 2024-11-12T12:24:22,427 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=03699d6939471fdc6521cd2a2d843827, ASSIGN; state=OFFLINE, location=4a91c05c96a4,38717,1731414257691; forceNewPlan=false, retain=false 2024-11-12T12:24:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T12:24:22,579 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=03699d6939471fdc6521cd2a2d843827, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:22,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=03699d6939471fdc6521cd2a2d843827, ASSIGN because future has completed 2024-11-12T12:24:22,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:24:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T12:24:22,745 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:22,745 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:24:22,746 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,746 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:24:22,746 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,746 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,750 INFO [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,753 INFO [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03699d6939471fdc6521cd2a2d843827 columnFamilyName A 2024-11-12T12:24:22,753 DEBUG [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:22,754 INFO [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] regionserver.HStore(327): Store=03699d6939471fdc6521cd2a2d843827/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:22,754 INFO [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,757 INFO [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03699d6939471fdc6521cd2a2d843827 columnFamilyName B 2024-11-12T12:24:22,757 DEBUG [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:22,758 INFO [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] regionserver.HStore(327): Store=03699d6939471fdc6521cd2a2d843827/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:22,759 INFO [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,761 INFO [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03699d6939471fdc6521cd2a2d843827 columnFamilyName C 2024-11-12T12:24:22,762 DEBUG [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:22,763 INFO [StoreOpener-03699d6939471fdc6521cd2a2d843827-1 {}] regionserver.HStore(327): Store=03699d6939471fdc6521cd2a2d843827/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:22,763 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,765 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,766 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,768 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,768 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,769 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:24:22,772 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,776 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T12:24:22,777 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 03699d6939471fdc6521cd2a2d843827; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66168495, jitterRate=-0.014012590050697327}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:24:22,778 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:22,779 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 03699d6939471fdc6521cd2a2d843827: Running coprocessor pre-open hook at 1731414262747Writing region info on filesystem at 1731414262747Initializing all the Stores at 1731414262749 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414262749Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414262749Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414262749Cleaning up temporary data from old regions at 1731414262768 (+19 ms)Running coprocessor post-open hooks at 1731414262778 (+10 ms)Region opened successfully at 1731414262779 (+1 ms) 2024-11-12T12:24:22,781 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., pid=6, masterSystemTime=1731414262738 2024-11-12T12:24:22,786 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:22,786 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:22,787 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=03699d6939471fdc6521cd2a2d843827, regionState=OPEN, openSeqNum=2, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:22,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:24:22,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T12:24:22,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 in 211 msec 2024-11-12T12:24:22,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T12:24:22,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03699d6939471fdc6521cd2a2d843827, ASSIGN in 379 msec 2024-11-12T12:24:22,808 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T12:24:22,809 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414262808"}]},"ts":"1731414262808"} 2024-11-12T12:24:22,813 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T12:24:22,815 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T12:24:22,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 552 msec 2024-11-12T12:24:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T12:24:22,954 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:22,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:22,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:22,960 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60daaef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:22,964 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bebe1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,967 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:22,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:22,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T12:24:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a1cfc3c 2024-11-12T12:24:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T12:24:22,982 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T12:24:22,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:22,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:22,989 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:22,994 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:22,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:22,998 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:23,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:23,004 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:23,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:23,011 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:23,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:23,016 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:23,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:23,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:23,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35159008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:23,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:23,030 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d96d2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:23,033 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8180ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:23,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:23,037 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14983026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:23,040 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a8a735f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:23,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:23,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b90827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:23,046 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc9366a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:23,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:23,050 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f11d0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:23,053 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:23,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b92dc16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:23,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:23,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83f8e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:23,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:23,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:23,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c944594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:23,064 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:23,064 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:23,064 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:23,065 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:23,066 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:23,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:23,067 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:23,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:23,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:23,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:23,069 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:23,069 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:23,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:23,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:23,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:23,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:23,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:24:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:24:23,079 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:24:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:24:23,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:24:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-12T12:24:23,090 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:23,092 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T12:24:23,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:24:23,095 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:23,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:23,097 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:23,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:23,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:23,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:23,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:24:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:24:23,113 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:23,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:24:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:23,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:23,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:24:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T12:24:23,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T12:24:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:23,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/facbb841fdfa45899924a1f220078511 is 50, key is test_row_1/A:col10/1731414263182/Put/seqid=0 2024-11-12T12:24:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414323350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414323351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414323356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414323363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414323386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T12:24:23,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741837_1013 (size=9657) 2024-11-12T12:24:23,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/facbb841fdfa45899924a1f220078511 2024-11-12T12:24:23,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T12:24:23,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:23,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:23,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:23,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414323493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414323493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414323494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414323494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414323512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/7cbc55b6f9d94b3e9f48f3235edabbbf is 50, key is test_row_1/B:col10/1731414263182/Put/seqid=0 2024-11-12T12:24:23,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741838_1014 (size=9657) 2024-11-12T12:24:23,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/7cbc55b6f9d94b3e9f48f3235edabbbf 2024-11-12T12:24:23,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T12:24:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:23,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/20cbb6b8595c4a3799b6898d97584488 is 50, key is test_row_1/C:col10/1731414263182/Put/seqid=0 2024-11-12T12:24:23,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741839_1015 (size=9657) 2024-11-12T12:24:23,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/20cbb6b8595c4a3799b6898d97584488 2024-11-12T12:24:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414323713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414323713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414323714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414323714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T12:24:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414323722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:23,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/facbb841fdfa45899924a1f220078511 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/facbb841fdfa45899924a1f220078511 2024-11-12T12:24:23,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/facbb841fdfa45899924a1f220078511, entries=100, sequenceid=13, filesize=9.4 K 2024-11-12T12:24:23,760 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-12T12:24:23,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/7cbc55b6f9d94b3e9f48f3235edabbbf as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7cbc55b6f9d94b3e9f48f3235edabbbf 2024-11-12T12:24:23,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7cbc55b6f9d94b3e9f48f3235edabbbf, entries=100, sequenceid=13, filesize=9.4 K 2024-11-12T12:24:23,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/20cbb6b8595c4a3799b6898d97584488 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/20cbb6b8595c4a3799b6898d97584488 2024-11-12T12:24:23,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T12:24:23,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:23,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:23,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:23,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:23,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/20cbb6b8595c4a3799b6898d97584488, entries=100, sequenceid=13, filesize=9.4 K 2024-11-12T12:24:23,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 03699d6939471fdc6521cd2a2d843827 in 613ms, sequenceid=13, compaction requested=false 2024-11-12T12:24:23,812 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T12:24:23,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:23,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T12:24:23,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:23,957 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:24:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/1fa6ba37d19d4ac09e7eca4c74f96110 is 50, key is test_row_0/A:col10/1731414263359/Put/seqid=0 2024-11-12T12:24:24,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741840_1016 (size=12001) 2024-11-12T12:24:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:24,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:24,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414324067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414324073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414324080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414324081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414324075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414324202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414324222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T12:24:24,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414324231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414324234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414324234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414324412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,429 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/1fa6ba37d19d4ac09e7eca4c74f96110 2024-11-12T12:24:24,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414324432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414324442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414324452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414324452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/00734457a1d5401b8e92b5ce37bef9ba is 50, key is test_row_0/B:col10/1731414263359/Put/seqid=0 2024-11-12T12:24:24,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741841_1017 (size=12001) 2024-11-12T12:24:24,519 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/00734457a1d5401b8e92b5ce37bef9ba 2024-11-12T12:24:24,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/fbcbcd881b02436db4aa6526b57d7e97 is 50, key is test_row_0/C:col10/1731414263359/Put/seqid=0 2024-11-12T12:24:24,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741842_1018 (size=12001) 2024-11-12T12:24:24,600 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/fbcbcd881b02436db4aa6526b57d7e97 2024-11-12T12:24:24,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/1fa6ba37d19d4ac09e7eca4c74f96110 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/1fa6ba37d19d4ac09e7eca4c74f96110 2024-11-12T12:24:24,646 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/1fa6ba37d19d4ac09e7eca4c74f96110, entries=150, sequenceid=38, filesize=11.7 K 2024-11-12T12:24:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/00734457a1d5401b8e92b5ce37bef9ba as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/00734457a1d5401b8e92b5ce37bef9ba 2024-11-12T12:24:24,670 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/00734457a1d5401b8e92b5ce37bef9ba, entries=150, sequenceid=38, filesize=11.7 K 2024-11-12T12:24:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/fbcbcd881b02436db4aa6526b57d7e97 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fbcbcd881b02436db4aa6526b57d7e97 2024-11-12T12:24:24,698 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fbcbcd881b02436db4aa6526b57d7e97, entries=150, sequenceid=38, filesize=11.7 K 2024-11-12T12:24:24,720 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 03699d6939471fdc6521cd2a2d843827 in 763ms, sequenceid=38, compaction requested=false 2024-11-12T12:24:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-12T12:24:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-12T12:24:24,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T12:24:24,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6250 sec 2024-11-12T12:24:24,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6580 sec 2024-11-12T12:24:24,744 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:24,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-12T12:24:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:24,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/018d18587a4040fea825f6fe8a103e4a is 50, key is test_row_0/A:col10/1731414264064/Put/seqid=0 2024-11-12T12:24:24,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741843_1019 (size=11997) 2024-11-12T12:24:24,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414324831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414324837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414324852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414324861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414324861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414324961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414324973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414324973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414324973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:24,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414324982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:24,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:24,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414325172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414325181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414325182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414325182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414325192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/018d18587a4040fea825f6fe8a103e4a 2024-11-12T12:24:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T12:24:25,242 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:25,243 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-12T12:24:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T12:24:25,248 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:25,251 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:25,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:25,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/7ff799b9c18b47beaf711db369f159e5 is 50, key is test_row_0/B:col10/1731414264064/Put/seqid=0 2024-11-12T12:24:25,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741844_1020 (size=9657) 2024-11-12T12:24:25,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/7ff799b9c18b47beaf711db369f159e5 2024-11-12T12:24:25,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/a23931d80db24b2d85e8b339538de841 is 50, key is test_row_0/C:col10/1731414264064/Put/seqid=0 2024-11-12T12:24:25,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741845_1021 (size=9657) 2024-11-12T12:24:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T12:24:25,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T12:24:25,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:25,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:25,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414325484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414325495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414325502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414325502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414325504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T12:24:25,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T12:24:25,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:25,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T12:24:25,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:25,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:25,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:25,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/a23931d80db24b2d85e8b339538de841 2024-11-12T12:24:25,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/018d18587a4040fea825f6fe8a103e4a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/018d18587a4040fea825f6fe8a103e4a 2024-11-12T12:24:25,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/018d18587a4040fea825f6fe8a103e4a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T12:24:25,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/7ff799b9c18b47beaf711db369f159e5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7ff799b9c18b47beaf711db369f159e5 2024-11-12T12:24:25,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7ff799b9c18b47beaf711db369f159e5, entries=100, sequenceid=50, filesize=9.4 K 2024-11-12T12:24:25,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/a23931d80db24b2d85e8b339538de841 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a23931d80db24b2d85e8b339538de841 2024-11-12T12:24:25,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a23931d80db24b2d85e8b339538de841, entries=100, sequenceid=50, filesize=9.4 K 2024-11-12T12:24:25,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 03699d6939471fdc6521cd2a2d843827 in 1056ms, sequenceid=50, compaction requested=true 2024-11-12T12:24:25,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:25,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:25,817 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:25,824 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:25,824 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:25,827 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:25,828 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:25,828 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:25,829 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,829 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/facbb841fdfa45899924a1f220078511, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/1fa6ba37d19d4ac09e7eca4c74f96110, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/018d18587a4040fea825f6fe8a103e4a] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=32.9 K 2024-11-12T12:24:25,830 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,831 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7cbc55b6f9d94b3e9f48f3235edabbbf, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/00734457a1d5401b8e92b5ce37bef9ba, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7ff799b9c18b47beaf711db369f159e5] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=30.6 K 2024-11-12T12:24:25,831 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting facbb841fdfa45899924a1f220078511, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731414263179 2024-11-12T12:24:25,833 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cbc55b6f9d94b3e9f48f3235edabbbf, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731414263179 2024-11-12T12:24:25,833 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa6ba37d19d4ac09e7eca4c74f96110, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731414263336 2024-11-12T12:24:25,834 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 018d18587a4040fea825f6fe8a103e4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731414264064 2024-11-12T12:24:25,837 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 00734457a1d5401b8e92b5ce37bef9ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731414263336 2024-11-12T12:24:25,838 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ff799b9c18b47beaf711db369f159e5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731414264064 2024-11-12T12:24:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T12:24:25,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T12:24:25,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,877 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:24:25,883 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#9 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:25,884 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/779306dd4e3249ad96bba33b44142d91 is 50, key is test_row_0/B:col10/1731414264064/Put/seqid=0 2024-11-12T12:24:25,890 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#10 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:25,891 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/c89d4a2eed84455ba1f4d3a7d55e30a0 is 50, key is test_row_0/A:col10/1731414264064/Put/seqid=0 2024-11-12T12:24:25,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/5e4b5c82bec040daa66179278853d6f9 is 50, key is test_row_0/A:col10/1731414264843/Put/seqid=0 2024-11-12T12:24:25,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741846_1022 (size=12104) 2024-11-12T12:24:25,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741847_1023 (size=12104) 2024-11-12T12:24:25,930 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/779306dd4e3249ad96bba33b44142d91 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/779306dd4e3249ad96bba33b44142d91 2024-11-12T12:24:25,944 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/c89d4a2eed84455ba1f4d3a7d55e30a0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c89d4a2eed84455ba1f4d3a7d55e30a0 2024-11-12T12:24:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741848_1024 (size=12001) 2024-11-12T12:24:25,970 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into 779306dd4e3249ad96bba33b44142d91(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:25,970 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:25,972 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into c89d4a2eed84455ba1f4d3a7d55e30a0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:25,972 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:25,974 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=13, startTime=1731414265815; duration=0sec 2024-11-12T12:24:25,974 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=13, startTime=1731414265802; duration=0sec 2024-11-12T12:24:25,974 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:25,975 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:25,975 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:25,976 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:25,977 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:25,977 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:25,977 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:25,978 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:25,978 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/20cbb6b8595c4a3799b6898d97584488, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fbcbcd881b02436db4aa6526b57d7e97, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a23931d80db24b2d85e8b339538de841] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=30.6 K 2024-11-12T12:24:25,979 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 20cbb6b8595c4a3799b6898d97584488, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731414263179 2024-11-12T12:24:25,980 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting fbcbcd881b02436db4aa6526b57d7e97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731414263336 2024-11-12T12:24:25,981 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a23931d80db24b2d85e8b339538de841, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731414264064 2024-11-12T12:24:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:26,042 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:26,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414326033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,042 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/027f889137a0492c82ee6a5bd54e96b6 is 50, key is test_row_0/C:col10/1731414264064/Put/seqid=0 2024-11-12T12:24:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414326035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414326037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414326038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414326043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741849_1025 (size=12104) 2024-11-12T12:24:26,100 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/027f889137a0492c82ee6a5bd54e96b6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/027f889137a0492c82ee6a5bd54e96b6 2024-11-12T12:24:26,118 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into 027f889137a0492c82ee6a5bd54e96b6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:26,118 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:26,118 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=13, startTime=1731414265815; duration=0sec 2024-11-12T12:24:26,118 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:26,118 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:26,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414326152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414326153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414326153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414326153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414326162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,364 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/5e4b5c82bec040daa66179278853d6f9 2024-11-12T12:24:26,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414326372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414326372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T12:24:26,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414326373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414326373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414326373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/f223111409b646c0b0b384b53202a046 is 50, key is test_row_0/B:col10/1731414264843/Put/seqid=0 2024-11-12T12:24:26,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741850_1026 (size=12001) 2024-11-12T12:24:26,422 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/f223111409b646c0b0b384b53202a046 2024-11-12T12:24:26,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/a3195be70a744c979a41d20bcee0cb13 is 50, key is test_row_0/C:col10/1731414264843/Put/seqid=0 2024-11-12T12:24:26,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741851_1027 (size=12001) 2024-11-12T12:24:26,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414326681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414326682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414326691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414326692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414326692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:26,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:26,907 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/a3195be70a744c979a41d20bcee0cb13 2024-11-12T12:24:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/5e4b5c82bec040daa66179278853d6f9 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5e4b5c82bec040daa66179278853d6f9 2024-11-12T12:24:26,938 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5e4b5c82bec040daa66179278853d6f9, entries=150, sequenceid=75, filesize=11.7 K 2024-11-12T12:24:26,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/f223111409b646c0b0b384b53202a046 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f223111409b646c0b0b384b53202a046 2024-11-12T12:24:26,954 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f223111409b646c0b0b384b53202a046, entries=150, sequenceid=75, filesize=11.7 K 2024-11-12T12:24:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-12T12:24:26,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/a3195be70a744c979a41d20bcee0cb13 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a3195be70a744c979a41d20bcee0cb13 2024-11-12T12:24:26,976 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a3195be70a744c979a41d20bcee0cb13, entries=150, sequenceid=75, filesize=11.7 K 2024-11-12T12:24:26,978 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 03699d6939471fdc6521cd2a2d843827 in 1101ms, sequenceid=75, compaction requested=false 2024-11-12T12:24:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-12T12:24:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-12T12:24:26,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T12:24:26,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7300 sec 2024-11-12T12:24:26,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7450 sec 2024-11-12T12:24:27,141 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T12:24:27,197 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:27,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-12T12:24:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:27,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/7f0440db09574651ab79db9c06da4f91 is 50, key is test_row_0/A:col10/1731414266036/Put/seqid=0 2024-11-12T12:24:27,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741852_1028 (size=14341) 2024-11-12T12:24:27,259 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T12:24:27,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414327265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414327270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414327271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414327274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414327277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,287 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T12:24:27,287 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T12:24:27,290 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T12:24:27,290 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T12:24:27,290 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T12:24:27,290 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T12:24:27,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T12:24:27,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T12:24:27,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414327386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414327387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T12:24:27,392 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:27,393 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-12T12:24:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T12:24:27,397 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:27,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414327392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,399 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:27,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414327392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414327393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T12:24:27,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T12:24:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:27,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:27,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:27,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414327592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414327601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414327613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414327613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414327613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/7f0440db09574651ab79db9c06da4f91 2024-11-12T12:24:27,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/33010ec21e5f4f0fb806c9bb9693c204 is 50, key is test_row_0/B:col10/1731414266036/Put/seqid=0 2024-11-12T12:24:27,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741853_1029 (size=12001) 2024-11-12T12:24:27,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T12:24:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:27,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:27,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:27,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/33010ec21e5f4f0fb806c9bb9693c204 2024-11-12T12:24:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T12:24:27,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/bd68dc502dc74a1db66a618fc5e479c7 is 50, key is test_row_0/C:col10/1731414266036/Put/seqid=0 2024-11-12T12:24:27,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741854_1030 (size=12001) 2024-11-12T12:24:27,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/bd68dc502dc74a1db66a618fc5e479c7 2024-11-12T12:24:27,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/7f0440db09574651ab79db9c06da4f91 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/7f0440db09574651ab79db9c06da4f91 2024-11-12T12:24:27,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/7f0440db09574651ab79db9c06da4f91, entries=200, sequenceid=91, filesize=14.0 K 2024-11-12T12:24:27,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/33010ec21e5f4f0fb806c9bb9693c204 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/33010ec21e5f4f0fb806c9bb9693c204 2024-11-12T12:24:27,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/33010ec21e5f4f0fb806c9bb9693c204, entries=150, sequenceid=91, filesize=11.7 K 2024-11-12T12:24:27,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/bd68dc502dc74a1db66a618fc5e479c7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/bd68dc502dc74a1db66a618fc5e479c7 2024-11-12T12:24:27,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/bd68dc502dc74a1db66a618fc5e479c7, entries=150, sequenceid=91, filesize=11.7 K 2024-11-12T12:24:27,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 03699d6939471fdc6521cd2a2d843827 in 632ms, sequenceid=91, compaction requested=true 2024-11-12T12:24:27,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:27,830 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:27,830 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:27,832 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:27,833 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:27,833 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:27,833 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:27,833 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:27,833 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:27,833 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c89d4a2eed84455ba1f4d3a7d55e30a0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5e4b5c82bec040daa66179278853d6f9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/7f0440db09574651ab79db9c06da4f91] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=37.5 K 2024-11-12T12:24:27,833 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/779306dd4e3249ad96bba33b44142d91, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f223111409b646c0b0b384b53202a046, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/33010ec21e5f4f0fb806c9bb9693c204] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=35.3 K 2024-11-12T12:24:27,834 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting c89d4a2eed84455ba1f4d3a7d55e30a0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731414263336 2024-11-12T12:24:27,835 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e4b5c82bec040daa66179278853d6f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731414264832 2024-11-12T12:24:27,836 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 779306dd4e3249ad96bba33b44142d91, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731414263336 2024-11-12T12:24:27,837 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f0440db09574651ab79db9c06da4f91, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731414266027 2024-11-12T12:24:27,837 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting f223111409b646c0b0b384b53202a046, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731414264832 2024-11-12T12:24:27,839 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 33010ec21e5f4f0fb806c9bb9693c204, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731414266027 2024-11-12T12:24:27,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T12:24:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:27,866 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-12T12:24:27,875 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#18 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:27,875 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/44549df3b14243e78b1bf8e05a858506 is 50, key is test_row_0/B:col10/1731414266036/Put/seqid=0 2024-11-12T12:24:27,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/75dc256688744c3c831dd22f116dcf7a is 50, key is test_row_0/A:col10/1731414267269/Put/seqid=0 2024-11-12T12:24:27,887 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:27,888 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/a81cbc13c9e44616bad14940bcc31c03 is 50, key is test_row_0/A:col10/1731414266036/Put/seqid=0 2024-11-12T12:24:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:27,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:27,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741855_1031 (size=12207) 2024-11-12T12:24:27,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414327945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741856_1032 (size=12001) 2024-11-12T12:24:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414327945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414327949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,955 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/75dc256688744c3c831dd22f116dcf7a 2024-11-12T12:24:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414327953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414327953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:27,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741857_1033 (size=12207) 2024-11-12T12:24:27,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/a08f07145d5440eba237833a50c242c3 is 50, key is test_row_0/B:col10/1731414267269/Put/seqid=0 2024-11-12T12:24:27,989 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/a81cbc13c9e44616bad14940bcc31c03 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/a81cbc13c9e44616bad14940bcc31c03 2024-11-12T12:24:28,006 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into a81cbc13c9e44616bad14940bcc31c03(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:28,006 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:28,006 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=13, startTime=1731414267830; duration=0sec 2024-11-12T12:24:28,006 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:28,007 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:28,007 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:28,009 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:28,009 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:28,009 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:28,009 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/027f889137a0492c82ee6a5bd54e96b6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a3195be70a744c979a41d20bcee0cb13, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/bd68dc502dc74a1db66a618fc5e479c7] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=35.3 K 2024-11-12T12:24:28,010 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027f889137a0492c82ee6a5bd54e96b6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731414263336 2024-11-12T12:24:28,013 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3195be70a744c979a41d20bcee0cb13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731414264832 2024-11-12T12:24:28,014 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd68dc502dc74a1db66a618fc5e479c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731414266027 2024-11-12T12:24:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T12:24:28,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741858_1034 (size=12001) 2024-11-12T12:24:28,025 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/a08f07145d5440eba237833a50c242c3 2024-11-12T12:24:28,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/d7666ae285474aeda6a6983e681cb023 is 50, key is test_row_0/C:col10/1731414267269/Put/seqid=0 2024-11-12T12:24:28,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741859_1035 (size=12001) 2024-11-12T12:24:28,060 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#23 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:28,062 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/d7666ae285474aeda6a6983e681cb023 2024-11-12T12:24:28,064 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/3867eff0c91645c9b2387f7b24020d46 is 50, key is test_row_0/C:col10/1731414266036/Put/seqid=0 2024-11-12T12:24:28,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414328062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414328062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414328062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414328072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414328072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741860_1036 (size=12207) 2024-11-12T12:24:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/75dc256688744c3c831dd22f116dcf7a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/75dc256688744c3c831dd22f116dcf7a 2024-11-12T12:24:28,091 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/75dc256688744c3c831dd22f116dcf7a, entries=150, sequenceid=114, filesize=11.7 K 2024-11-12T12:24:28,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/a08f07145d5440eba237833a50c242c3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/a08f07145d5440eba237833a50c242c3 2024-11-12T12:24:28,097 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/3867eff0c91645c9b2387f7b24020d46 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3867eff0c91645c9b2387f7b24020d46 2024-11-12T12:24:28,108 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/a08f07145d5440eba237833a50c242c3, entries=150, sequenceid=114, filesize=11.7 K 2024-11-12T12:24:28,115 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into 3867eff0c91645c9b2387f7b24020d46(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:28,115 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:28,115 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=13, startTime=1731414267830; duration=0sec 2024-11-12T12:24:28,115 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:28,115 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:28,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/d7666ae285474aeda6a6983e681cb023 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d7666ae285474aeda6a6983e681cb023 2024-11-12T12:24:28,137 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d7666ae285474aeda6a6983e681cb023, entries=150, sequenceid=114, filesize=11.7 K 2024-11-12T12:24:28,138 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 03699d6939471fdc6521cd2a2d843827 in 273ms, sequenceid=114, compaction requested=false 2024-11-12T12:24:28,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:28,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-12T12:24:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-12T12:24:28,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T12:24:28,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 747 msec 2024-11-12T12:24:28,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 758 msec 2024-11-12T12:24:28,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:28,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-12T12:24:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:28,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/5adc760820624aa29a55bb852a28e952 is 50, key is test_row_0/A:col10/1731414268284/Put/seqid=0 2024-11-12T12:24:28,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741861_1037 (size=16781) 2024-11-12T12:24:28,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/5adc760820624aa29a55bb852a28e952 2024-11-12T12:24:28,345 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/44549df3b14243e78b1bf8e05a858506 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/44549df3b14243e78b1bf8e05a858506 2024-11-12T12:24:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/2768685ab11d41fbb1c92f2b55b02070 is 50, key is test_row_0/B:col10/1731414268284/Put/seqid=0 2024-11-12T12:24:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414328337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414328338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414328338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414328343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414328347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741862_1038 (size=12101) 2024-11-12T12:24:28,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/2768685ab11d41fbb1c92f2b55b02070 2024-11-12T12:24:28,378 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into 44549df3b14243e78b1bf8e05a858506(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:28,378 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:28,378 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=13, startTime=1731414267830; duration=0sec 2024-11-12T12:24:28,378 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:28,378 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:28,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/d172e63aa18e4d55aed3e59049456283 is 50, key is test_row_0/C:col10/1731414268284/Put/seqid=0 2024-11-12T12:24:28,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741863_1039 (size=12101) 2024-11-12T12:24:28,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414328452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414328462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414328462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414328463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414328471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T12:24:28,533 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:28,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-12T12:24:28,543 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T12:24:28,545 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:28,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T12:24:28,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414328661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414328671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414328672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414328681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414328677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:28,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:28,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:28,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T12:24:28,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:28,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:28,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:28,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:28,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:28,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/d172e63aa18e4d55aed3e59049456283 2024-11-12T12:24:28,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/5adc760820624aa29a55bb852a28e952 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5adc760820624aa29a55bb852a28e952 2024-11-12T12:24:28,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5adc760820624aa29a55bb852a28e952, entries=250, sequenceid=132, filesize=16.4 K 2024-11-12T12:24:28,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/2768685ab11d41fbb1c92f2b55b02070 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2768685ab11d41fbb1c92f2b55b02070 2024-11-12T12:24:28,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T12:24:28,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:28,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:28,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:28,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:28,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T12:24:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:28,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2768685ab11d41fbb1c92f2b55b02070, entries=150, sequenceid=132, filesize=11.8 K 2024-11-12T12:24:28,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/d172e63aa18e4d55aed3e59049456283 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d172e63aa18e4d55aed3e59049456283 2024-11-12T12:24:28,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d172e63aa18e4d55aed3e59049456283, entries=150, sequenceid=132, filesize=11.8 K 2024-11-12T12:24:28,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 03699d6939471fdc6521cd2a2d843827 in 605ms, sequenceid=132, compaction requested=true 2024-11-12T12:24:28,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:28,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:28,893 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:28,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:28,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:28,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:28,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:28,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:28,893 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:28,900 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:28,900 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:28,901 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:28,901 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/a81cbc13c9e44616bad14940bcc31c03, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/75dc256688744c3c831dd22f116dcf7a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5adc760820624aa29a55bb852a28e952] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=40.0 K 2024-11-12T12:24:28,907 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:28,907 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a81cbc13c9e44616bad14940bcc31c03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731414266027 2024-11-12T12:24:28,907 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:28,908 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:28,908 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/44549df3b14243e78b1bf8e05a858506, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/a08f07145d5440eba237833a50c242c3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2768685ab11d41fbb1c92f2b55b02070] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=35.5 K 2024-11-12T12:24:28,911 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44549df3b14243e78b1bf8e05a858506, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731414266027 2024-11-12T12:24:28,911 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 75dc256688744c3c831dd22f116dcf7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731414267265 2024-11-12T12:24:28,918 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 5adc760820624aa29a55bb852a28e952, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731414267926 2024-11-12T12:24:28,918 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08f07145d5440eba237833a50c242c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731414267265 2024-11-12T12:24:28,919 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2768685ab11d41fbb1c92f2b55b02070, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731414267949 2024-11-12T12:24:28,946 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:28,947 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/bddb7a77ae9f44d1a97623dc9e8db8fd is 50, key is test_row_0/A:col10/1731414268284/Put/seqid=0 2024-11-12T12:24:28,959 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:28,960 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/226600a973a04b7a8a0ce7fb6dd9e502 is 50, key is test_row_0/B:col10/1731414268284/Put/seqid=0 2024-11-12T12:24:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:28,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:28,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-12T12:24:28,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741864_1040 (size=12409) 2024-11-12T12:24:29,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/da5700a0da8541ba8554665c553af4e5 is 50, key is test_row_0/A:col10/1731414268337/Put/seqid=0 2024-11-12T12:24:29,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741865_1041 (size=12409) 2024-11-12T12:24:29,015 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/bddb7a77ae9f44d1a97623dc9e8db8fd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bddb7a77ae9f44d1a97623dc9e8db8fd 2024-11-12T12:24:29,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T12:24:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741866_1042 (size=14541) 2024-11-12T12:24:29,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/da5700a0da8541ba8554665c553af4e5 2024-11-12T12:24:29,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/9174394c5ae94b8f93b31eaf665b5f25 is 50, key is test_row_0/B:col10/1731414268337/Put/seqid=0 2024-11-12T12:24:29,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741867_1043 (size=12151) 2024-11-12T12:24:29,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/9174394c5ae94b8f93b31eaf665b5f25 2024-11-12T12:24:29,059 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into bddb7a77ae9f44d1a97623dc9e8db8fd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:29,059 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:29,059 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=13, startTime=1731414268893; duration=0sec 2024-11-12T12:24:29,060 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:29,060 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:29,060 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:29,064 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:29,064 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:29,064 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,065 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3867eff0c91645c9b2387f7b24020d46, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d7666ae285474aeda6a6983e681cb023, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d172e63aa18e4d55aed3e59049456283] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=35.5 K 2024-11-12T12:24:29,066 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3867eff0c91645c9b2387f7b24020d46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731414266027 2024-11-12T12:24:29,066 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d7666ae285474aeda6a6983e681cb023, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731414267265 2024-11-12T12:24:29,067 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d172e63aa18e4d55aed3e59049456283, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731414267949 2024-11-12T12:24:29,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/7adaeb1ca3494cddbc94fef4152fc297 is 50, key is test_row_0/C:col10/1731414268337/Put/seqid=0 2024-11-12T12:24:29,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414329079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414329081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414329082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414329086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,099 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#32 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:29,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414329087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,100 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/8a8f258a2d3d4f439036ba0ae7a127c5 is 50, key is test_row_0/C:col10/1731414268284/Put/seqid=0 2024-11-12T12:24:29,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741868_1044 (size=12151) 2024-11-12T12:24:29,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741869_1045 (size=12409) 2024-11-12T12:24:29,135 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/8a8f258a2d3d4f439036ba0ae7a127c5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/8a8f258a2d3d4f439036ba0ae7a127c5 2024-11-12T12:24:29,148 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into 8a8f258a2d3d4f439036ba0ae7a127c5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:29,148 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:29,148 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=13, startTime=1731414268893; duration=0sec 2024-11-12T12:24:29,148 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:29,149 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T12:24:29,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T12:24:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414329192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414329192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414329202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414329202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414329211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T12:24:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414329402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414329402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414329411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414329421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,425 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/226600a973a04b7a8a0ce7fb6dd9e502 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/226600a973a04b7a8a0ce7fb6dd9e502 2024-11-12T12:24:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414329422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,437 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into 226600a973a04b7a8a0ce7fb6dd9e502(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:29,437 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:29,437 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=13, startTime=1731414268893; duration=0sec 2024-11-12T12:24:29,438 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:29,438 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:29,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T12:24:29,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:29,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:29,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/7adaeb1ca3494cddbc94fef4152fc297 2024-11-12T12:24:29,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/da5700a0da8541ba8554665c553af4e5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/da5700a0da8541ba8554665c553af4e5 2024-11-12T12:24:29,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/da5700a0da8541ba8554665c553af4e5, entries=200, sequenceid=154, filesize=14.2 K 2024-11-12T12:24:29,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/9174394c5ae94b8f93b31eaf665b5f25 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/9174394c5ae94b8f93b31eaf665b5f25 2024-11-12T12:24:29,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/9174394c5ae94b8f93b31eaf665b5f25, entries=150, sequenceid=154, filesize=11.9 K 2024-11-12T12:24:29,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/7adaeb1ca3494cddbc94fef4152fc297 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7adaeb1ca3494cddbc94fef4152fc297 2024-11-12T12:24:29,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7adaeb1ca3494cddbc94fef4152fc297, entries=150, sequenceid=154, filesize=11.9 K 2024-11-12T12:24:29,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 03699d6939471fdc6521cd2a2d843827 in 609ms, sequenceid=154, compaction requested=false 2024-11-12T12:24:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:29,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T12:24:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:29,645 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-12T12:24:29,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/6cf35ce6e63c4cb1a399f4a2b3ce85ca is 50, key is test_row_0/A:col10/1731414269078/Put/seqid=0 2024-11-12T12:24:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T12:24:29,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741870_1046 (size=12151) 2024-11-12T12:24:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:29,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:29,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414329752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414329754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414329758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414329762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414329762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414329871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414329871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414329871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414329871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:29,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414329881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414330082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414330082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414330083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414330083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414330092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,113 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/6cf35ce6e63c4cb1a399f4a2b3ce85ca 2024-11-12T12:24:30,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/2541ba74f9734060a8f04641509cc9bd is 50, key is test_row_0/B:col10/1731414269078/Put/seqid=0 2024-11-12T12:24:30,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741871_1047 (size=12151) 2024-11-12T12:24:30,189 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/2541ba74f9734060a8f04641509cc9bd 2024-11-12T12:24:30,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/b9edb9c47f184b76929f6d912367eb06 is 50, key is test_row_0/C:col10/1731414269078/Put/seqid=0 2024-11-12T12:24:30,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741872_1048 (size=12151) 2024-11-12T12:24:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414330392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414330395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414330401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414330402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414330402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,635 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/b9edb9c47f184b76929f6d912367eb06 2024-11-12T12:24:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/6cf35ce6e63c4cb1a399f4a2b3ce85ca as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/6cf35ce6e63c4cb1a399f4a2b3ce85ca 2024-11-12T12:24:30,660 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/6cf35ce6e63c4cb1a399f4a2b3ce85ca, entries=150, sequenceid=172, filesize=11.9 K 2024-11-12T12:24:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/2541ba74f9734060a8f04641509cc9bd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2541ba74f9734060a8f04641509cc9bd 2024-11-12T12:24:30,677 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2541ba74f9734060a8f04641509cc9bd, entries=150, sequenceid=172, filesize=11.9 K 2024-11-12T12:24:30,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/b9edb9c47f184b76929f6d912367eb06 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b9edb9c47f184b76929f6d912367eb06 2024-11-12T12:24:30,687 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b9edb9c47f184b76929f6d912367eb06, entries=150, sequenceid=172, filesize=11.9 K 2024-11-12T12:24:30,688 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 03699d6939471fdc6521cd2a2d843827 in 1043ms, sequenceid=172, compaction requested=true 2024-11-12T12:24:30,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:30,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:30,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-12T12:24:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-12T12:24:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T12:24:30,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T12:24:30,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1450 sec 2024-11-12T12:24:30,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1680 sec 2024-11-12T12:24:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:30,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-12T12:24:30,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/3171a59752e1444d9feed738a7d6c029 is 50, key is test_row_0/A:col10/1731414270904/Put/seqid=0 2024-11-12T12:24:30,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741873_1049 (size=14541) 2024-11-12T12:24:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414330921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/3171a59752e1444d9feed738a7d6c029 2024-11-12T12:24:30,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414330925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414330925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414330927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414330928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:30,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/54f062f85d3e4bc2a62f7d81136c38fd is 50, key is test_row_0/B:col10/1731414270904/Put/seqid=0 2024-11-12T12:24:30,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741874_1050 (size=12151) 2024-11-12T12:24:31,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/54f062f85d3e4bc2a62f7d81136c38fd 2024-11-12T12:24:31,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/077b1e496725411287e7ab9d4815d335 is 50, key is test_row_0/C:col10/1731414270904/Put/seqid=0 2024-11-12T12:24:31,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741875_1051 (size=12151) 2024-11-12T12:24:31,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/077b1e496725411287e7ab9d4815d335 2024-11-12T12:24:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414331041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414331041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414331041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414331042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/3171a59752e1444d9feed738a7d6c029 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/3171a59752e1444d9feed738a7d6c029 2024-11-12T12:24:31,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/3171a59752e1444d9feed738a7d6c029, entries=200, sequenceid=196, filesize=14.2 K 2024-11-12T12:24:31,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/54f062f85d3e4bc2a62f7d81136c38fd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/54f062f85d3e4bc2a62f7d81136c38fd 2024-11-12T12:24:31,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/54f062f85d3e4bc2a62f7d81136c38fd, entries=150, sequenceid=196, filesize=11.9 K 2024-11-12T12:24:31,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/077b1e496725411287e7ab9d4815d335 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/077b1e496725411287e7ab9d4815d335 2024-11-12T12:24:31,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/077b1e496725411287e7ab9d4815d335, entries=150, sequenceid=196, filesize=11.9 K 2024-11-12T12:24:31,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 03699d6939471fdc6521cd2a2d843827 in 183ms, sequenceid=196, compaction requested=true 2024-11-12T12:24:31,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:31,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:31,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:31,088 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:24:31,088 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:24:31,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:31,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:31,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:31,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:31,090 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53642 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:24:31,090 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:24:31,090 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:31,090 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:31,090 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:31,090 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:31,090 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/226600a973a04b7a8a0ce7fb6dd9e502, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/9174394c5ae94b8f93b31eaf665b5f25, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2541ba74f9734060a8f04641509cc9bd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/54f062f85d3e4bc2a62f7d81136c38fd] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=47.7 K 2024-11-12T12:24:31,090 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bddb7a77ae9f44d1a97623dc9e8db8fd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/da5700a0da8541ba8554665c553af4e5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/6cf35ce6e63c4cb1a399f4a2b3ce85ca, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/3171a59752e1444d9feed738a7d6c029] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=52.4 K 2024-11-12T12:24:31,091 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 226600a973a04b7a8a0ce7fb6dd9e502, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731414267949 2024-11-12T12:24:31,091 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting bddb7a77ae9f44d1a97623dc9e8db8fd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731414267949 2024-11-12T12:24:31,092 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 9174394c5ae94b8f93b31eaf665b5f25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731414268337 2024-11-12T12:24:31,092 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting da5700a0da8541ba8554665c553af4e5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731414268326 2024-11-12T12:24:31,092 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2541ba74f9734060a8f04641509cc9bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731414268995 2024-11-12T12:24:31,092 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf35ce6e63c4cb1a399f4a2b3ce85ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731414268995 2024-11-12T12:24:31,096 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3171a59752e1444d9feed738a7d6c029, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731414269746 2024-11-12T12:24:31,096 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f062f85d3e4bc2a62f7d81136c38fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731414269758 2024-11-12T12:24:31,115 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:31,116 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:31,116 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/d43edf058d264b51ab2a70ea057ee147 is 50, key is test_row_0/B:col10/1731414270904/Put/seqid=0 2024-11-12T12:24:31,116 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/c5636f2281e843eb9fe2232ee0823bcf is 50, key is test_row_0/A:col10/1731414270904/Put/seqid=0 2024-11-12T12:24:31,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741876_1052 (size=12595) 2024-11-12T12:24:31,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741877_1053 (size=12595) 2024-11-12T12:24:31,153 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/c5636f2281e843eb9fe2232ee0823bcf as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c5636f2281e843eb9fe2232ee0823bcf 2024-11-12T12:24:31,163 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into c5636f2281e843eb9fe2232ee0823bcf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:31,163 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:31,163 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=12, startTime=1731414271088; duration=0sec 2024-11-12T12:24:31,163 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:31,163 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:31,163 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:24:31,165 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:24:31,165 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:31,165 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:31,166 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/8a8f258a2d3d4f439036ba0ae7a127c5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7adaeb1ca3494cddbc94fef4152fc297, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b9edb9c47f184b76929f6d912367eb06, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/077b1e496725411287e7ab9d4815d335] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=47.7 K 2024-11-12T12:24:31,167 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a8f258a2d3d4f439036ba0ae7a127c5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731414267949 2024-11-12T12:24:31,167 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7adaeb1ca3494cddbc94fef4152fc297, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731414268337 2024-11-12T12:24:31,168 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9edb9c47f184b76929f6d912367eb06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731414268995 2024-11-12T12:24:31,168 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 077b1e496725411287e7ab9d4815d335, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731414269758 2024-11-12T12:24:31,183 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:31,183 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/b5aec0b0c75549b7a2305a87cf15e9b1 is 50, key is test_row_0/C:col10/1731414270904/Put/seqid=0 2024-11-12T12:24:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741878_1054 (size=12595) 2024-11-12T12:24:31,198 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/b5aec0b0c75549b7a2305a87cf15e9b1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b5aec0b0c75549b7a2305a87cf15e9b1 2024-11-12T12:24:31,210 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into b5aec0b0c75549b7a2305a87cf15e9b1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:31,210 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:31,210 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=12, startTime=1731414271088; duration=0sec 2024-11-12T12:24:31,210 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:31,210 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:31,255 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:31,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-12T12:24:31,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/500e029ab5224e5d9e8d07824eb9703d is 50, key is test_row_0/A:col10/1731414271254/Put/seqid=0 2024-11-12T12:24:31,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741879_1055 (size=14541) 2024-11-12T12:24:31,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414331328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414331329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414331329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414331329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414331441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414331442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414331441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414331442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,547 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/d43edf058d264b51ab2a70ea057ee147 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d43edf058d264b51ab2a70ea057ee147 2024-11-12T12:24:31,562 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into d43edf058d264b51ab2a70ea057ee147(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:31,563 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:31,563 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=12, startTime=1731414271088; duration=0sec 2024-11-12T12:24:31,563 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:31,563 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:31,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414331652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414331652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414331652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414331652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/500e029ab5224e5d9e8d07824eb9703d 2024-11-12T12:24:31,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/0f0ff4d60f744da189c6db26da53b472 is 50, key is test_row_0/B:col10/1731414271254/Put/seqid=0 2024-11-12T12:24:31,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741880_1056 (size=12151) 2024-11-12T12:24:31,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/0f0ff4d60f744da189c6db26da53b472 2024-11-12T12:24:31,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/1619b644aba1450a98ae8087cb6fa7d4 is 50, key is test_row_0/C:col10/1731414271254/Put/seqid=0 2024-11-12T12:24:31,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741881_1057 (size=12151) 2024-11-12T12:24:31,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/1619b644aba1450a98ae8087cb6fa7d4 2024-11-12T12:24:31,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/500e029ab5224e5d9e8d07824eb9703d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/500e029ab5224e5d9e8d07824eb9703d 2024-11-12T12:24:31,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/500e029ab5224e5d9e8d07824eb9703d, entries=200, sequenceid=211, filesize=14.2 K 2024-11-12T12:24:31,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/0f0ff4d60f744da189c6db26da53b472 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/0f0ff4d60f744da189c6db26da53b472 2024-11-12T12:24:31,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/0f0ff4d60f744da189c6db26da53b472, entries=150, sequenceid=211, filesize=11.9 K 2024-11-12T12:24:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/1619b644aba1450a98ae8087cb6fa7d4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/1619b644aba1450a98ae8087cb6fa7d4 2024-11-12T12:24:31,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/1619b644aba1450a98ae8087cb6fa7d4, entries=150, sequenceid=211, filesize=11.9 K 2024-11-12T12:24:31,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 03699d6939471fdc6521cd2a2d843827 in 625ms, sequenceid=211, compaction requested=false 2024-11-12T12:24:31,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:31,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:31,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-12T12:24:31,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/c25b567e603d452ba9bd4f5d3dbd563c is 50, key is test_row_0/A:col10/1731414271327/Put/seqid=0 2024-11-12T12:24:31,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741882_1058 (size=14541) 2024-11-12T12:24:31,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414331973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414331976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414331976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414331992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414331997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414332092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414332092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414332101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414332101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414332101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414332301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414332301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414332311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414332312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414332312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/c25b567e603d452ba9bd4f5d3dbd563c 2024-11-12T12:24:32,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/92ae11d8bd05409eaf2221de13d00709 is 50, key is test_row_0/B:col10/1731414271327/Put/seqid=0 2024-11-12T12:24:32,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741883_1059 (size=12151) 2024-11-12T12:24:32,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/92ae11d8bd05409eaf2221de13d00709 2024-11-12T12:24:32,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/3339eda7a14f4156a4fec4f4bb9c0379 is 50, key is test_row_0/C:col10/1731414271327/Put/seqid=0 2024-11-12T12:24:32,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741884_1060 (size=12151) 2024-11-12T12:24:32,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414332611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414332611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414332622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414332622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414332623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T12:24:32,712 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:32,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-12T12:24:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T12:24:32,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:32,718 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:32,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T12:24:32,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/3339eda7a14f4156a4fec4f4bb9c0379 2024-11-12T12:24:32,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/c25b567e603d452ba9bd4f5d3dbd563c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c25b567e603d452ba9bd4f5d3dbd563c 2024-11-12T12:24:32,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T12:24:32,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:32,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:32,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:32,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:32,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:32,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c25b567e603d452ba9bd4f5d3dbd563c, entries=200, sequenceid=236, filesize=14.2 K 2024-11-12T12:24:32,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/92ae11d8bd05409eaf2221de13d00709 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/92ae11d8bd05409eaf2221de13d00709 2024-11-12T12:24:32,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/92ae11d8bd05409eaf2221de13d00709, entries=150, sequenceid=236, filesize=11.9 K 2024-11-12T12:24:32,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/3339eda7a14f4156a4fec4f4bb9c0379 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3339eda7a14f4156a4fec4f4bb9c0379 2024-11-12T12:24:32,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3339eda7a14f4156a4fec4f4bb9c0379, entries=150, sequenceid=236, filesize=11.9 K 2024-11-12T12:24:32,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 03699d6939471fdc6521cd2a2d843827 in 978ms, sequenceid=236, compaction requested=true 2024-11-12T12:24:32,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:32,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:32,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:32,933 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:32,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:32,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:32,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:32,935 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:32,936 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:32,936 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:32,936 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:32,936 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:32,936 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:32,936 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:32,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T12:24:32,936 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c5636f2281e843eb9fe2232ee0823bcf, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/500e029ab5224e5d9e8d07824eb9703d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c25b567e603d452ba9bd4f5d3dbd563c] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=40.7 K 2024-11-12T12:24:32,936 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d43edf058d264b51ab2a70ea057ee147, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/0f0ff4d60f744da189c6db26da53b472, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/92ae11d8bd05409eaf2221de13d00709] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=36.0 K 2024-11-12T12:24:32,937 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d43edf058d264b51ab2a70ea057ee147, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731414269758 2024-11-12T12:24:32,937 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5636f2281e843eb9fe2232ee0823bcf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731414269758 2024-11-12T12:24:32,938 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f0ff4d60f744da189c6db26da53b472, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731414270924 2024-11-12T12:24:32,938 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 500e029ab5224e5d9e8d07824eb9703d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731414270922 2024-11-12T12:24:32,938 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ae11d8bd05409eaf2221de13d00709, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731414271290 2024-11-12T12:24:32,938 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting c25b567e603d452ba9bd4f5d3dbd563c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731414271290 2024-11-12T12:24:32,974 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:32,975 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/d0f9237e8e0c4a048e21a3cbcb85b7ec is 50, key is test_row_0/B:col10/1731414271327/Put/seqid=0 2024-11-12T12:24:32,979 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:32,979 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/5b33cc625a454b5e8002dd12d7c51733 is 50, key is test_row_0/A:col10/1731414271327/Put/seqid=0 2024-11-12T12:24:33,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741886_1062 (size=12697) 2024-11-12T12:24:33,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741885_1061 (size=12697) 2024-11-12T12:24:33,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T12:24:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:33,029 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-12T12:24:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T12:24:33,032 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/5b33cc625a454b5e8002dd12d7c51733 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5b33cc625a454b5e8002dd12d7c51733 2024-11-12T12:24:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/07f50c80f67049f3a8207daa10431574 is 50, key is test_row_0/A:col10/1731414271963/Put/seqid=0 2024-11-12T12:24:33,046 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into 5b33cc625a454b5e8002dd12d7c51733(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:33,046 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:33,046 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=13, startTime=1731414272933; duration=0sec 2024-11-12T12:24:33,046 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:33,046 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:33,046 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:33,051 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:33,051 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:33,051 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:33,051 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b5aec0b0c75549b7a2305a87cf15e9b1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/1619b644aba1450a98ae8087cb6fa7d4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3339eda7a14f4156a4fec4f4bb9c0379] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=36.0 K 2024-11-12T12:24:33,052 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5aec0b0c75549b7a2305a87cf15e9b1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731414269758 2024-11-12T12:24:33,052 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1619b644aba1450a98ae8087cb6fa7d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731414270924 2024-11-12T12:24:33,053 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3339eda7a14f4156a4fec4f4bb9c0379, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731414271290 2024-11-12T12:24:33,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741887_1063 (size=12151) 2024-11-12T12:24:33,074 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:33,075 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/c05f7e6b3c784a22b8567fcd80c933a7 is 50, key is test_row_0/C:col10/1731414271327/Put/seqid=0 2024-11-12T12:24:33,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741888_1064 (size=12697) 2024-11-12T12:24:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:33,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:33,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414333173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414333178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414333178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414333180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414333180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414333291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414333292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414333292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414333292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414333292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T12:24:33,430 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/d0f9237e8e0c4a048e21a3cbcb85b7ec as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d0f9237e8e0c4a048e21a3cbcb85b7ec 2024-11-12T12:24:33,444 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into d0f9237e8e0c4a048e21a3cbcb85b7ec(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:33,444 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:33,444 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=13, startTime=1731414272933; duration=0sec 2024-11-12T12:24:33,445 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:33,445 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:33,471 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/07f50c80f67049f3a8207daa10431574 2024-11-12T12:24:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/28ad977d815a451a96aa8badcae0dcdd is 50, key is test_row_0/B:col10/1731414271963/Put/seqid=0 2024-11-12T12:24:33,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414333501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414333501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741889_1065 (size=12151) 2024-11-12T12:24:33,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414333502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414333502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414333502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,514 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/c05f7e6b3c784a22b8567fcd80c933a7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c05f7e6b3c784a22b8567fcd80c933a7 2024-11-12T12:24:33,524 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into c05f7e6b3c784a22b8567fcd80c933a7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:33,524 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:33,524 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=13, startTime=1731414272934; duration=0sec 2024-11-12T12:24:33,524 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:33,524 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:33,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414333811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414333811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414333811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414333812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414333821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T12:24:33,907 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/28ad977d815a451a96aa8badcae0dcdd 2024-11-12T12:24:33,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/ef7506695e0340fa9d224340aa0d9d75 is 50, key is test_row_0/C:col10/1731414271963/Put/seqid=0 2024-11-12T12:24:33,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741890_1066 (size=12151) 2024-11-12T12:24:34,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414334322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:34,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414334322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:34,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:34,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:34,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414334322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:34,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414334322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:34,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:34,325 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/ef7506695e0340fa9d224340aa0d9d75 2024-11-12T12:24:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:34,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414334332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/07f50c80f67049f3a8207daa10431574 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/07f50c80f67049f3a8207daa10431574 2024-11-12T12:24:34,341 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/07f50c80f67049f3a8207daa10431574, entries=150, sequenceid=249, filesize=11.9 K 2024-11-12T12:24:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/28ad977d815a451a96aa8badcae0dcdd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/28ad977d815a451a96aa8badcae0dcdd 2024-11-12T12:24:34,348 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/28ad977d815a451a96aa8badcae0dcdd, entries=150, sequenceid=249, filesize=11.9 K 2024-11-12T12:24:34,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/ef7506695e0340fa9d224340aa0d9d75 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ef7506695e0340fa9d224340aa0d9d75 2024-11-12T12:24:34,359 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ef7506695e0340fa9d224340aa0d9d75, entries=150, sequenceid=249, filesize=11.9 K 2024-11-12T12:24:34,361 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 03699d6939471fdc6521cd2a2d843827 in 1332ms, sequenceid=249, compaction requested=false 2024-11-12T12:24:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-12T12:24:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-12T12:24:34,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T12:24:34,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6450 sec 2024-11-12T12:24:34,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6560 sec 2024-11-12T12:24:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T12:24:34,861 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:34,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-12T12:24:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T12:24:34,869 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:34,871 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:34,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T12:24:35,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T12:24:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:35,027 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-12T12:24:35,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/14181751edb4485f9309380ab515e217 is 50, key is test_row_0/A:col10/1731414273177/Put/seqid=0 2024-11-12T12:24:35,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741891_1067 (size=12301) 2024-11-12T12:24:35,063 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/14181751edb4485f9309380ab515e217 2024-11-12T12:24:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/20ca4628f1bc4db7b655c211f1e3dde1 is 50, key is test_row_0/B:col10/1731414273177/Put/seqid=0 2024-11-12T12:24:35,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741892_1068 (size=12301) 2024-11-12T12:24:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T12:24:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:35,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:35,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414335347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414335349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414335349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414335350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414335351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,358 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in 03699d6939471fdc6521cd2a2d843827 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:24:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414335461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414335461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414335461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414335461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,495 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/20ca4628f1bc4db7b655c211f1e3dde1 2024-11-12T12:24:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T12:24:35,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/57598079a7e643b9893cd7c77972e94b is 50, key is test_row_0/C:col10/1731414273177/Put/seqid=0 2024-11-12T12:24:35,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741893_1069 (size=12301) 2024-11-12T12:24:35,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414335671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414335673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414335673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414335681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:35,911 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/57598079a7e643b9893cd7c77972e94b 2024-11-12T12:24:35,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/14181751edb4485f9309380ab515e217 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/14181751edb4485f9309380ab515e217 2024-11-12T12:24:35,922 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/14181751edb4485f9309380ab515e217, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T12:24:35,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/20ca4628f1bc4db7b655c211f1e3dde1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/20ca4628f1bc4db7b655c211f1e3dde1 2024-11-12T12:24:35,928 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/20ca4628f1bc4db7b655c211f1e3dde1, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T12:24:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/57598079a7e643b9893cd7c77972e94b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/57598079a7e643b9893cd7c77972e94b 2024-11-12T12:24:35,936 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/57598079a7e643b9893cd7c77972e94b, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T12:24:35,937 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 03699d6939471fdc6521cd2a2d843827 in 911ms, sequenceid=275, compaction requested=true 2024-11-12T12:24:35,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:35,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:35,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-12T12:24:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-12T12:24:35,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T12:24:35,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0680 sec 2024-11-12T12:24:35,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0800 sec 2024-11-12T12:24:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:35,986 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:35,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-12T12:24:35,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/38e3648d50b443bc903a28804e34566d is 50, key is test_row_0/A:col10/1731414275345/Put/seqid=0 2024-11-12T12:24:36,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741894_1070 (size=12301) 2024-11-12T12:24:36,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/38e3648d50b443bc903a28804e34566d 2024-11-12T12:24:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T12:24:36,012 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:36,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-12T12:24:36,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/6f402edecb2144a1ba1bcc8122e8348f is 50, key is test_row_0/B:col10/1731414275345/Put/seqid=0 2024-11-12T12:24:36,016 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T12:24:36,017 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:36,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:36,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741895_1071 (size=12301) 2024-11-12T12:24:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414336024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414336026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414336032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414336034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T12:24:36,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414336141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414336141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414336141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414336141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T12:24:36,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:36,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:36,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:36,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T12:24:36,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:36,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:36,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T12:24:36,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414336351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414336351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414336351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414336351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/6f402edecb2144a1ba1bcc8122e8348f 2024-11-12T12:24:36,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/7b837eada3664dfca48632663d56dc44 is 50, key is test_row_0/C:col10/1731414275345/Put/seqid=0 2024-11-12T12:24:36,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741896_1072 (size=12301) 2024-11-12T12:24:36,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/7b837eada3664dfca48632663d56dc44 2024-11-12T12:24:36,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/38e3648d50b443bc903a28804e34566d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/38e3648d50b443bc903a28804e34566d 2024-11-12T12:24:36,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/38e3648d50b443bc903a28804e34566d, entries=150, sequenceid=290, filesize=12.0 K 2024-11-12T12:24:36,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/6f402edecb2144a1ba1bcc8122e8348f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6f402edecb2144a1ba1bcc8122e8348f 2024-11-12T12:24:36,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6f402edecb2144a1ba1bcc8122e8348f, entries=150, sequenceid=290, filesize=12.0 K 2024-11-12T12:24:36,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/7b837eada3664dfca48632663d56dc44 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7b837eada3664dfca48632663d56dc44 2024-11-12T12:24:36,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7b837eada3664dfca48632663d56dc44, entries=150, sequenceid=290, filesize=12.0 K 2024-11-12T12:24:36,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T12:24:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:36,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 03699d6939471fdc6521cd2a2d843827 in 493ms, sequenceid=290, compaction requested=true 2024-11-12T12:24:36,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:36,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:36,480 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:24:36,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:36,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:36,480 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:24:36,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:36,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:36,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:36,483 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:24:36,483 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:36,483 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,483 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:24:36,484 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:36,484 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,484 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5b33cc625a454b5e8002dd12d7c51733, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/07f50c80f67049f3a8207daa10431574, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/14181751edb4485f9309380ab515e217, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/38e3648d50b443bc903a28804e34566d] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=48.3 K 2024-11-12T12:24:36,484 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d0f9237e8e0c4a048e21a3cbcb85b7ec, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/28ad977d815a451a96aa8badcae0dcdd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/20ca4628f1bc4db7b655c211f1e3dde1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6f402edecb2144a1ba1bcc8122e8348f] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=48.3 K 2024-11-12T12:24:36,484 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b33cc625a454b5e8002dd12d7c51733, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731414271290 2024-11-12T12:24:36,485 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f50c80f67049f3a8207daa10431574, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731414271963 2024-11-12T12:24:36,485 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f9237e8e0c4a048e21a3cbcb85b7ec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731414271290 2024-11-12T12:24:36,485 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14181751edb4485f9309380ab515e217, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731414273175 2024-11-12T12:24:36,485 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ad977d815a451a96aa8badcae0dcdd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731414271963 2024-11-12T12:24:36,486 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38e3648d50b443bc903a28804e34566d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414275345 2024-11-12T12:24:36,486 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 20ca4628f1bc4db7b655c211f1e3dde1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731414273175 2024-11-12T12:24:36,486 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f402edecb2144a1ba1bcc8122e8348f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414275345 2024-11-12T12:24:36,502 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#60 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:36,502 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#61 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:36,502 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/f59ce7ef14a346f5933c55a54b91ad26 is 50, key is test_row_0/A:col10/1731414275345/Put/seqid=0 2024-11-12T12:24:36,502 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/46fcb01708374dc1b51ee6a609bb7471 is 50, key is test_row_0/B:col10/1731414275345/Put/seqid=0 2024-11-12T12:24:36,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741897_1073 (size=12983) 2024-11-12T12:24:36,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741898_1074 (size=12983) 2024-11-12T12:24:36,524 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/46fcb01708374dc1b51ee6a609bb7471 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/46fcb01708374dc1b51ee6a609bb7471 2024-11-12T12:24:36,532 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into 46fcb01708374dc1b51ee6a609bb7471(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:36,532 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:36,532 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=12, startTime=1731414276480; duration=0sec 2024-11-12T12:24:36,532 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:36,532 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:36,532 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:24:36,534 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:24:36,534 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:36,534 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,535 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c05f7e6b3c784a22b8567fcd80c933a7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ef7506695e0340fa9d224340aa0d9d75, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/57598079a7e643b9893cd7c77972e94b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7b837eada3664dfca48632663d56dc44] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=48.3 K 2024-11-12T12:24:36,535 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting c05f7e6b3c784a22b8567fcd80c933a7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731414271290 2024-11-12T12:24:36,535 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ef7506695e0340fa9d224340aa0d9d75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731414271963 2024-11-12T12:24:36,536 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 57598079a7e643b9893cd7c77972e94b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731414273175 2024-11-12T12:24:36,537 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b837eada3664dfca48632663d56dc44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414275345 2024-11-12T12:24:36,550 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:36,551 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/a07c48f3b6ec414b8e5bc18ccc06a2ab is 50, key is test_row_0/C:col10/1731414275345/Put/seqid=0 2024-11-12T12:24:36,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741899_1075 (size=12983) 2024-11-12T12:24:36,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T12:24:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,633 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-12T12:24:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/cc6d7994f5ec4d08a49d554b26508e7a is 50, key is test_row_0/A:col10/1731414276020/Put/seqid=0 2024-11-12T12:24:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T12:24:36,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741900_1076 (size=12301) 2024-11-12T12:24:36,647 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/cc6d7994f5ec4d08a49d554b26508e7a 2024-11-12T12:24:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/885c2016f16a4dbdbfe595a03f893693 is 50, key is test_row_0/B:col10/1731414276020/Put/seqid=0 2024-11-12T12:24:36,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741901_1077 (size=12301) 2024-11-12T12:24:36,661 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/885c2016f16a4dbdbfe595a03f893693 2024-11-12T12:24:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:36,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:36,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/505d981999914fe9917bd7b59484e81b is 50, key is test_row_0/C:col10/1731414276020/Put/seqid=0 2024-11-12T12:24:36,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741902_1078 (size=12301) 2024-11-12T12:24:36,676 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/505d981999914fe9917bd7b59484e81b 2024-11-12T12:24:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/cc6d7994f5ec4d08a49d554b26508e7a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/cc6d7994f5ec4d08a49d554b26508e7a 2024-11-12T12:24:36,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414336684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414336685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,689 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/cc6d7994f5ec4d08a49d554b26508e7a, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T12:24:36,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/885c2016f16a4dbdbfe595a03f893693 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/885c2016f16a4dbdbfe595a03f893693 2024-11-12T12:24:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414336686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414336686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,696 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/885c2016f16a4dbdbfe595a03f893693, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T12:24:36,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/505d981999914fe9917bd7b59484e81b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/505d981999914fe9917bd7b59484e81b 2024-11-12T12:24:36,705 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/505d981999914fe9917bd7b59484e81b, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T12:24:36,707 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 03699d6939471fdc6521cd2a2d843827 in 73ms, sequenceid=312, compaction requested=false 2024-11-12T12:24:36,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:36,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:36,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-12T12:24:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-12T12:24:36,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-12T12:24:36,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 691 msec 2024-11-12T12:24:36,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 699 msec 2024-11-12T12:24:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:36,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:36,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-12T12:24:36,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/ec428c42f20b4145ac04be259bd94fc1 is 50, key is test_row_0/A:col10/1731414276794/Put/seqid=0 2024-11-12T12:24:36,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741903_1079 (size=17181) 2024-11-12T12:24:36,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/ec428c42f20b4145ac04be259bd94fc1 2024-11-12T12:24:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414336824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414336828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414336831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414336832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/b239d9395c054be3aec813e0fa8a2b05 is 50, key is test_row_0/B:col10/1731414276794/Put/seqid=0 2024-11-12T12:24:36,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741904_1080 (size=12301) 2024-11-12T12:24:36,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/b239d9395c054be3aec813e0fa8a2b05 2024-11-12T12:24:36,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/13970f8e64d94c0cba361a352908f3bc is 50, key is test_row_0/C:col10/1731414276794/Put/seqid=0 2024-11-12T12:24:36,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741905_1081 (size=12301) 2024-11-12T12:24:36,923 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/f59ce7ef14a346f5933c55a54b91ad26 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f59ce7ef14a346f5933c55a54b91ad26 2024-11-12T12:24:36,929 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into f59ce7ef14a346f5933c55a54b91ad26(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:36,929 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:36,929 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=12, startTime=1731414276480; duration=0sec 2024-11-12T12:24:36,929 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:36,929 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414336941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414336941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414336941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414336941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:36,961 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/a07c48f3b6ec414b8e5bc18ccc06a2ab as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a07c48f3b6ec414b8e5bc18ccc06a2ab 2024-11-12T12:24:36,972 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into a07c48f3b6ec414b8e5bc18ccc06a2ab(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:36,972 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:36,972 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=12, startTime=1731414276480; duration=0sec 2024-11-12T12:24:36,973 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:36,973 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T12:24:37,151 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:37,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414337151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414337151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414337151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-12T12:24:37,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414337151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T12:24:37,154 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:37,155 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:37,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T12:24:37,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/13970f8e64d94c0cba361a352908f3bc 2024-11-12T12:24:37,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/ec428c42f20b4145ac04be259bd94fc1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/ec428c42f20b4145ac04be259bd94fc1 2024-11-12T12:24:37,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/ec428c42f20b4145ac04be259bd94fc1, entries=250, sequenceid=329, filesize=16.8 K 2024-11-12T12:24:37,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/b239d9395c054be3aec813e0fa8a2b05 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b239d9395c054be3aec813e0fa8a2b05 2024-11-12T12:24:37,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b239d9395c054be3aec813e0fa8a2b05, entries=150, sequenceid=329, filesize=12.0 K 2024-11-12T12:24:37,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/13970f8e64d94c0cba361a352908f3bc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/13970f8e64d94c0cba361a352908f3bc 2024-11-12T12:24:37,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/13970f8e64d94c0cba361a352908f3bc, entries=150, sequenceid=329, filesize=12.0 K 2024-11-12T12:24:37,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 03699d6939471fdc6521cd2a2d843827 in 498ms, sequenceid=329, compaction requested=true 2024-11-12T12:24:37,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:37,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:37,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:37,294 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:37,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:37,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:37,294 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:37,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:37,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:37,295 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:37,295 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:37,295 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:37,295 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:37,295 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:37,295 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:37,295 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f59ce7ef14a346f5933c55a54b91ad26, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/cc6d7994f5ec4d08a49d554b26508e7a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/ec428c42f20b4145ac04be259bd94fc1] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=41.5 K 2024-11-12T12:24:37,295 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/46fcb01708374dc1b51ee6a609bb7471, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/885c2016f16a4dbdbfe595a03f893693, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b239d9395c054be3aec813e0fa8a2b05] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=36.7 K 2024-11-12T12:24:37,296 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting f59ce7ef14a346f5933c55a54b91ad26, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414275345 2024-11-12T12:24:37,296 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 46fcb01708374dc1b51ee6a609bb7471, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414275345 2024-11-12T12:24:37,296 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc6d7994f5ec4d08a49d554b26508e7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731414276020 2024-11-12T12:24:37,297 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 885c2016f16a4dbdbfe595a03f893693, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731414276020 2024-11-12T12:24:37,297 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting b239d9395c054be3aec813e0fa8a2b05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731414276685 2024-11-12T12:24:37,297 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec428c42f20b4145ac04be259bd94fc1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731414276681 2024-11-12T12:24:37,308 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:37,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T12:24:37,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:37,308 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/bf325a5c06004ecda0df36e8795c2479 is 50, key is test_row_0/A:col10/1731414276794/Put/seqid=0 2024-11-12T12:24:37,308 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-12T12:24:37,311 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:37,311 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/b791dc048a39415d8ec15b4e781f5264 is 50, key is test_row_0/B:col10/1731414276794/Put/seqid=0 2024-11-12T12:24:37,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/4ddf4a4c8d9d49259090171837ca7305 is 50, key is test_row_0/A:col10/1731414276819/Put/seqid=0 2024-11-12T12:24:37,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741907_1083 (size=13085) 2024-11-12T12:24:37,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741906_1082 (size=13085) 2024-11-12T12:24:37,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741908_1084 (size=12301) 2024-11-12T12:24:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:37,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:37,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414337426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414337462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414337462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414337462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414337462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T12:24:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414337541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,737 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/bf325a5c06004ecda0df36e8795c2479 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bf325a5c06004ecda0df36e8795c2479 2024-11-12T12:24:37,742 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/4ddf4a4c8d9d49259090171837ca7305 2024-11-12T12:24:37,744 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into bf325a5c06004ecda0df36e8795c2479(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:37,744 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:37,744 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=13, startTime=1731414277294; duration=0sec 2024-11-12T12:24:37,745 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:37,745 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:37,745 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:37,746 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/b791dc048a39415d8ec15b4e781f5264 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b791dc048a39415d8ec15b4e781f5264 2024-11-12T12:24:37,748 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:37,748 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:37,748 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:37,748 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a07c48f3b6ec414b8e5bc18ccc06a2ab, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/505d981999914fe9917bd7b59484e81b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/13970f8e64d94c0cba361a352908f3bc] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=36.7 K 2024-11-12T12:24:37,750 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a07c48f3b6ec414b8e5bc18ccc06a2ab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414275345 2024-11-12T12:24:37,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414337751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,753 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 505d981999914fe9917bd7b59484e81b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731414276020 2024-11-12T12:24:37,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,754 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13970f8e64d94c0cba361a352908f3bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731414276685 2024-11-12T12:24:37,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/38561494d13342839c5d9060a43762e0 is 50, key is test_row_0/B:col10/1731414276819/Put/seqid=0 2024-11-12T12:24:37,758 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into b791dc048a39415d8ec15b4e781f5264(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:37,758 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:37,758 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=13, startTime=1731414277294; duration=0sec 2024-11-12T12:24:37,758 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:37,758 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:37,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741909_1085 (size=12301) 2024-11-12T12:24:37,773 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:37,774 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/540ca5e8ea7a4f5aa21c8dce46b27fb4 is 50, key is test_row_0/C:col10/1731414276794/Put/seqid=0 2024-11-12T12:24:37,775 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/38561494d13342839c5d9060a43762e0 2024-11-12T12:24:37,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741910_1086 (size=13085) 2024-11-12T12:24:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T12:24:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/ac9011a1ebca4bcba763d12cb9f20596 is 50, key is test_row_0/C:col10/1731414276819/Put/seqid=0 2024-11-12T12:24:37,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741911_1087 (size=12301) 2024-11-12T12:24:37,795 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/ac9011a1ebca4bcba763d12cb9f20596 2024-11-12T12:24:37,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/4ddf4a4c8d9d49259090171837ca7305 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4ddf4a4c8d9d49259090171837ca7305 2024-11-12T12:24:37,810 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4ddf4a4c8d9d49259090171837ca7305, entries=150, sequenceid=351, filesize=12.0 K 2024-11-12T12:24:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/38561494d13342839c5d9060a43762e0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/38561494d13342839c5d9060a43762e0 2024-11-12T12:24:37,817 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/38561494d13342839c5d9060a43762e0, entries=150, sequenceid=351, filesize=12.0 K 2024-11-12T12:24:37,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/ac9011a1ebca4bcba763d12cb9f20596 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9011a1ebca4bcba763d12cb9f20596 2024-11-12T12:24:37,825 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9011a1ebca4bcba763d12cb9f20596, entries=150, sequenceid=351, filesize=12.0 K 2024-11-12T12:24:37,828 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 03699d6939471fdc6521cd2a2d843827 in 520ms, sequenceid=351, compaction requested=false 2024-11-12T12:24:37,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:37,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:37,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-12T12:24:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-12T12:24:37,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T12:24:37,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 674 msec 2024-11-12T12:24:37,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 681 msec 2024-11-12T12:24:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:37,973 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:37,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-12T12:24:37,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/aeee2ae450694a5091ccac132401fb6a is 50, key is test_row_0/A:col10/1731414277973/Put/seqid=0 2024-11-12T12:24:37,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741912_1088 (size=14741) 2024-11-12T12:24:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414337993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414337994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414337995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414337997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414338061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414338101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414338102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414338102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414338102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,187 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/540ca5e8ea7a4f5aa21c8dce46b27fb4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/540ca5e8ea7a4f5aa21c8dce46b27fb4 2024-11-12T12:24:38,193 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into 540ca5e8ea7a4f5aa21c8dce46b27fb4(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:38,193 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:38,193 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=13, startTime=1731414277294; duration=0sec 2024-11-12T12:24:38,193 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:38,193 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T12:24:38,292 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:38,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:38,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414338311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414338311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414338311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414338311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-12T12:24:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T12:24:38,317 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:38,318 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:38,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:38,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/aeee2ae450694a5091ccac132401fb6a 2024-11-12T12:24:38,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/1e3d01ad51e0409f8afedd6c65e1532f is 50, key is test_row_0/B:col10/1731414277973/Put/seqid=0 2024-11-12T12:24:38,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741913_1089 (size=12301) 2024-11-12T12:24:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T12:24:38,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T12:24:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:38,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414338571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414338621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T12:24:38,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414338621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:38,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414338621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:38,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414338621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T12:24:38,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T12:24:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:38,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/1e3d01ad51e0409f8afedd6c65e1532f 2024-11-12T12:24:38,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/681129b5a5944c438b225205526adfc6 is 50, key is test_row_0/C:col10/1731414277973/Put/seqid=0 2024-11-12T12:24:38,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741914_1090 (size=12301) 2024-11-12T12:24:38,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T12:24:38,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:38,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:38,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:38,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T12:24:39,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T12:24:39,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:39,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:39,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:39,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:39,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:39,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414339132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414339132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:39,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:39,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414339132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414339132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:39,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/681129b5a5944c438b225205526adfc6 2024-11-12T12:24:39,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/aeee2ae450694a5091ccac132401fb6a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/aeee2ae450694a5091ccac132401fb6a 2024-11-12T12:24:39,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/aeee2ae450694a5091ccac132401fb6a, entries=200, sequenceid=372, filesize=14.4 K 2024-11-12T12:24:39,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/1e3d01ad51e0409f8afedd6c65e1532f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/1e3d01ad51e0409f8afedd6c65e1532f 2024-11-12T12:24:39,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/1e3d01ad51e0409f8afedd6c65e1532f, entries=150, sequenceid=372, filesize=12.0 K 2024-11-12T12:24:39,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/681129b5a5944c438b225205526adfc6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/681129b5a5944c438b225205526adfc6 2024-11-12T12:24:39,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T12:24:39,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:39,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:39,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:39,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:39,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:39,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/681129b5a5944c438b225205526adfc6, entries=150, sequenceid=372, filesize=12.0 K 2024-11-12T12:24:39,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 03699d6939471fdc6521cd2a2d843827 in 1273ms, sequenceid=372, compaction requested=true 2024-11-12T12:24:39,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:39,246 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:39,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:39,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:39,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:39,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:39,247 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:39,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:39,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:39,247 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:39,247 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:39,248 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:39,248 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:39,248 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:39,248 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bf325a5c06004ecda0df36e8795c2479, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4ddf4a4c8d9d49259090171837ca7305, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/aeee2ae450694a5091ccac132401fb6a] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=39.2 K 2024-11-12T12:24:39,248 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:39,248 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b791dc048a39415d8ec15b4e781f5264, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/38561494d13342839c5d9060a43762e0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/1e3d01ad51e0409f8afedd6c65e1532f] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=36.8 K 2024-11-12T12:24:39,248 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf325a5c06004ecda0df36e8795c2479, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731414276685 2024-11-12T12:24:39,248 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting b791dc048a39415d8ec15b4e781f5264, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731414276685 2024-11-12T12:24:39,250 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ddf4a4c8d9d49259090171837ca7305, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731414276819 2024-11-12T12:24:39,251 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 38561494d13342839c5d9060a43762e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731414276819 2024-11-12T12:24:39,252 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e3d01ad51e0409f8afedd6c65e1532f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731414277417 2024-11-12T12:24:39,252 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeee2ae450694a5091ccac132401fb6a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731414277417 2024-11-12T12:24:39,267 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:39,267 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/614fe388f81741a6930c920a1ebf4b6e is 50, key is test_row_0/B:col10/1731414277973/Put/seqid=0 2024-11-12T12:24:39,269 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:39,270 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/4144609f64664449962fb4aedfdc8b5e is 50, key is test_row_0/A:col10/1731414277973/Put/seqid=0 2024-11-12T12:24:39,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741915_1091 (size=13187) 2024-11-12T12:24:39,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741916_1092 (size=13187) 2024-11-12T12:24:39,292 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/614fe388f81741a6930c920a1ebf4b6e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/614fe388f81741a6930c920a1ebf4b6e 2024-11-12T12:24:39,298 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/4144609f64664449962fb4aedfdc8b5e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4144609f64664449962fb4aedfdc8b5e 2024-11-12T12:24:39,307 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into 614fe388f81741a6930c920a1ebf4b6e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:39,307 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:39,307 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=13, startTime=1731414279246; duration=0sec 2024-11-12T12:24:39,308 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:39,308 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:39,308 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:39,309 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:39,309 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into 4144609f64664449962fb4aedfdc8b5e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:39,309 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:39,309 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:39,309 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=13, startTime=1731414279246; duration=0sec 2024-11-12T12:24:39,310 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:39,310 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/540ca5e8ea7a4f5aa21c8dce46b27fb4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9011a1ebca4bcba763d12cb9f20596, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/681129b5a5944c438b225205526adfc6] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=36.8 K 2024-11-12T12:24:39,310 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:39,310 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:39,310 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 540ca5e8ea7a4f5aa21c8dce46b27fb4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731414276685 2024-11-12T12:24:39,311 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ac9011a1ebca4bcba763d12cb9f20596, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731414276819 2024-11-12T12:24:39,311 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 681129b5a5944c438b225205526adfc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731414277417 2024-11-12T12:24:39,319 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:39,320 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/baea77712f3f40cfad7767550f5d81b5 is 50, key is test_row_0/C:col10/1731414277973/Put/seqid=0 2024-11-12T12:24:39,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741917_1093 (size=13187) 2024-11-12T12:24:39,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T12:24:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:39,392 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-12T12:24:39,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/9beeca6134d9434097e2282f17226699 is 50, key is test_row_0/A:col10/1731414277984/Put/seqid=0 2024-11-12T12:24:39,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741918_1094 (size=12301) 2024-11-12T12:24:39,402 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/9beeca6134d9434097e2282f17226699 2024-11-12T12:24:39,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/2efeb23abc50426ab940ddd8be90ba01 is 50, key is test_row_0/B:col10/1731414277984/Put/seqid=0 2024-11-12T12:24:39,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741919_1095 (size=12301) 2024-11-12T12:24:39,440 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/2efeb23abc50426ab940ddd8be90ba01 2024-11-12T12:24:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T12:24:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/aec900831d40479dbb5ea181afa2f033 is 50, key is test_row_0/C:col10/1731414277984/Put/seqid=0 2024-11-12T12:24:39,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741920_1096 (size=12301) 2024-11-12T12:24:39,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:39,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414339666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:39,731 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/baea77712f3f40cfad7767550f5d81b5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/baea77712f3f40cfad7767550f5d81b5 2024-11-12T12:24:39,745 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into baea77712f3f40cfad7767550f5d81b5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:39,745 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:39,745 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=13, startTime=1731414279247; duration=0sec 2024-11-12T12:24:39,745 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:39,745 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:39,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414339771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:39,867 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/aec900831d40479dbb5ea181afa2f033 2024-11-12T12:24:39,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/9beeca6134d9434097e2282f17226699 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/9beeca6134d9434097e2282f17226699 2024-11-12T12:24:39,879 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/9beeca6134d9434097e2282f17226699, entries=150, sequenceid=392, filesize=12.0 K 2024-11-12T12:24:39,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/2efeb23abc50426ab940ddd8be90ba01 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2efeb23abc50426ab940ddd8be90ba01 2024-11-12T12:24:39,895 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2efeb23abc50426ab940ddd8be90ba01, entries=150, sequenceid=392, filesize=12.0 K 2024-11-12T12:24:39,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/aec900831d40479dbb5ea181afa2f033 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/aec900831d40479dbb5ea181afa2f033 2024-11-12T12:24:39,904 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/aec900831d40479dbb5ea181afa2f033, entries=150, sequenceid=392, filesize=12.0 K 2024-11-12T12:24:39,905 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 03699d6939471fdc6521cd2a2d843827 in 513ms, sequenceid=392, compaction requested=false 2024-11-12T12:24:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-12T12:24:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-12T12:24:39,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-12T12:24:39,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5890 sec 2024-11-12T12:24:39,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6170 sec 2024-11-12T12:24:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:39,983 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:39,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-12T12:24:39,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/f7c7110adedd42e384b923fe0b00765c is 50, key is test_row_0/A:col10/1731414279664/Put/seqid=0 2024-11-12T12:24:40,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741921_1097 (size=14741) 2024-11-12T12:24:40,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414340020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:40,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414340131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:40,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414340141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:40,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414340141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:40,143 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 03699d6939471fdc6521cd2a2d843827 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:24:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:40,145 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 03699d6939471fdc6521cd2a2d843827 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:24:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:40,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414340151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:40,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414340152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:40,155 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in 03699d6939471fdc6521cd2a2d843827 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:24:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:40,156 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 03699d6939471fdc6521cd2a2d843827 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:24:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:40,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414340341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:40,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/f7c7110adedd42e384b923fe0b00765c 2024-11-12T12:24:40,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/19bc9744295a4787ad287da7c471ce8b is 50, key is test_row_0/B:col10/1731414279664/Put/seqid=0 2024-11-12T12:24:40,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741922_1098 (size=12301) 2024-11-12T12:24:40,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/19bc9744295a4787ad287da7c471ce8b 2024-11-12T12:24:40,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/0ccf0c5d99254876ac0c590a43b37836 is 50, key is test_row_0/C:col10/1731414279664/Put/seqid=0 2024-11-12T12:24:40,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741923_1099 (size=12301) 2024-11-12T12:24:40,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/0ccf0c5d99254876ac0c590a43b37836 2024-11-12T12:24:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T12:24:40,472 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:40,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-12T12:24:40,475 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T12:24:40,477 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:40,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:40,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/f7c7110adedd42e384b923fe0b00765c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f7c7110adedd42e384b923fe0b00765c 2024-11-12T12:24:40,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f7c7110adedd42e384b923fe0b00765c, entries=200, sequenceid=413, filesize=14.4 K 2024-11-12T12:24:40,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/19bc9744295a4787ad287da7c471ce8b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/19bc9744295a4787ad287da7c471ce8b 2024-11-12T12:24:40,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/19bc9744295a4787ad287da7c471ce8b, entries=150, sequenceid=413, filesize=12.0 K 2024-11-12T12:24:40,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/0ccf0c5d99254876ac0c590a43b37836 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/0ccf0c5d99254876ac0c590a43b37836 2024-11-12T12:24:40,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/0ccf0c5d99254876ac0c590a43b37836, entries=150, sequenceid=413, filesize=12.0 K 2024-11-12T12:24:40,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 03699d6939471fdc6521cd2a2d843827 in 534ms, sequenceid=413, compaction requested=true 2024-11-12T12:24:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:40,518 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:40,518 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:40,519 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:40,519 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:40,519 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:40,519 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/614fe388f81741a6930c920a1ebf4b6e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2efeb23abc50426ab940ddd8be90ba01, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/19bc9744295a4787ad287da7c471ce8b] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=36.9 K 2024-11-12T12:24:40,520 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:40,520 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:40,520 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:40,520 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4144609f64664449962fb4aedfdc8b5e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/9beeca6134d9434097e2282f17226699, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f7c7110adedd42e384b923fe0b00765c] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=39.3 K 2024-11-12T12:24:40,520 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 614fe388f81741a6930c920a1ebf4b6e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731414277417 2024-11-12T12:24:40,520 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4144609f64664449962fb4aedfdc8b5e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731414277417 2024-11-12T12:24:40,521 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efeb23abc50426ab940ddd8be90ba01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731414277984 2024-11-12T12:24:40,521 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9beeca6134d9434097e2282f17226699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731414277984 2024-11-12T12:24:40,521 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 19bc9744295a4787ad287da7c471ce8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731414279659 2024-11-12T12:24:40,522 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7c7110adedd42e384b923fe0b00765c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731414279653 2024-11-12T12:24:40,540 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:40,541 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/065c088cff6b4ad9865cf34cb82315e3 is 50, key is test_row_0/A:col10/1731414279664/Put/seqid=0 2024-11-12T12:24:40,547 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:40,548 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/c9ad1693de524288a28f16cdc7a9d032 is 50, key is test_row_0/B:col10/1731414279664/Put/seqid=0 2024-11-12T12:24:40,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741924_1100 (size=13289) 2024-11-12T12:24:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T12:24:40,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741925_1101 (size=13289) 2024-11-12T12:24:40,602 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/c9ad1693de524288a28f16cdc7a9d032 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/c9ad1693de524288a28f16cdc7a9d032 2024-11-12T12:24:40,613 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into c9ad1693de524288a28f16cdc7a9d032(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:40,613 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:40,613 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=13, startTime=1731414280518; duration=0sec 2024-11-12T12:24:40,614 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:40,614 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:40,614 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:40,615 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:40,615 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:40,615 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:40,615 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/baea77712f3f40cfad7767550f5d81b5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/aec900831d40479dbb5ea181afa2f033, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/0ccf0c5d99254876ac0c590a43b37836] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=36.9 K 2024-11-12T12:24:40,616 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting baea77712f3f40cfad7767550f5d81b5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731414277417 2024-11-12T12:24:40,617 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting aec900831d40479dbb5ea181afa2f033, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731414277984 2024-11-12T12:24:40,617 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ccf0c5d99254876ac0c590a43b37836, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731414279659 2024-11-12T12:24:40,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-12T12:24:40,635 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#89 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:40,635 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-12T12:24:40,636 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/fd7a03ba4dd049b1b75ab8d2728867aa is 50, key is test_row_0/C:col10/1731414279664/Put/seqid=0 2024-11-12T12:24:40,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/dd5982a0f2784dc3814cbcedf2ed6201 is 50, key is test_row_0/A:col10/1731414280018/Put/seqid=0 2024-11-12T12:24:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:40,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:40,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741926_1102 (size=13289) 2024-11-12T12:24:40,679 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/fd7a03ba4dd049b1b75ab8d2728867aa as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fd7a03ba4dd049b1b75ab8d2728867aa 2024-11-12T12:24:40,689 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into fd7a03ba4dd049b1b75ab8d2728867aa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:40,689 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:40,689 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=13, startTime=1731414280518; duration=0sec 2024-11-12T12:24:40,689 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:40,689 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:40,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741927_1103 (size=12301) 2024-11-12T12:24:40,697 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/dd5982a0f2784dc3814cbcedf2ed6201 2024-11-12T12:24:40,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/360a964f5047497ca79c500525981289 is 50, key is test_row_0/B:col10/1731414280018/Put/seqid=0 2024-11-12T12:24:40,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741928_1104 (size=12301) 2024-11-12T12:24:40,747 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/360a964f5047497ca79c500525981289 2024-11-12T12:24:40,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/c18d49dccfc145d7aa269115e4028919 is 50, key is test_row_0/C:col10/1731414280018/Put/seqid=0 2024-11-12T12:24:40,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741929_1105 (size=12301) 2024-11-12T12:24:40,787 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/c18d49dccfc145d7aa269115e4028919 2024-11-12T12:24:40,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T12:24:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414340786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:40,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/dd5982a0f2784dc3814cbcedf2ed6201 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/dd5982a0f2784dc3814cbcedf2ed6201 2024-11-12T12:24:40,816 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/dd5982a0f2784dc3814cbcedf2ed6201, entries=150, sequenceid=430, filesize=12.0 K 2024-11-12T12:24:40,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/360a964f5047497ca79c500525981289 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/360a964f5047497ca79c500525981289 2024-11-12T12:24:40,825 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/360a964f5047497ca79c500525981289, entries=150, sequenceid=430, filesize=12.0 K 2024-11-12T12:24:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/c18d49dccfc145d7aa269115e4028919 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c18d49dccfc145d7aa269115e4028919 2024-11-12T12:24:40,838 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c18d49dccfc145d7aa269115e4028919, entries=150, sequenceid=430, filesize=12.0 K 2024-11-12T12:24:40,839 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 03699d6939471fdc6521cd2a2d843827 in 204ms, sequenceid=430, compaction requested=false 2024-11-12T12:24:40,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:40,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:40,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-12T12:24:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-12T12:24:40,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-12T12:24:40,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 364 msec 2024-11-12T12:24:40,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 372 msec 2024-11-12T12:24:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:40,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:40,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-12T12:24:40,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/fddfd29f5e7744aba986c93576c9a272 is 50, key is test_row_0/A:col10/1731414280780/Put/seqid=0 2024-11-12T12:24:40,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741930_1106 (size=14741) 2024-11-12T12:24:40,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/fddfd29f5e7744aba986c93576c9a272 2024-11-12T12:24:40,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/6fc7075bc49643058135aac0f94cc23b is 50, key is test_row_0/B:col10/1731414280780/Put/seqid=0 2024-11-12T12:24:40,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741931_1107 (size=12301) 2024-11-12T12:24:40,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/6fc7075bc49643058135aac0f94cc23b 2024-11-12T12:24:40,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/d99648a30f5b4b12b9792b03639557a2 is 50, key is test_row_0/C:col10/1731414280780/Put/seqid=0 2024-11-12T12:24:40,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:40,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741932_1108 (size=12301) 2024-11-12T12:24:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414340974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:40,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/d99648a30f5b4b12b9792b03639557a2 2024-11-12T12:24:40,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/fddfd29f5e7744aba986c93576c9a272 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/fddfd29f5e7744aba986c93576c9a272 2024-11-12T12:24:40,989 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/065c088cff6b4ad9865cf34cb82315e3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/065c088cff6b4ad9865cf34cb82315e3 2024-11-12T12:24:40,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/fddfd29f5e7744aba986c93576c9a272, entries=200, sequenceid=452, filesize=14.4 K 2024-11-12T12:24:40,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/6fc7075bc49643058135aac0f94cc23b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6fc7075bc49643058135aac0f94cc23b 2024-11-12T12:24:41,001 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into 065c088cff6b4ad9865cf34cb82315e3(size=13.0 K), total size for store is 39.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:41,002 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:41,002 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=13, startTime=1731414280518; duration=0sec 2024-11-12T12:24:41,002 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:41,002 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:41,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6fc7075bc49643058135aac0f94cc23b, entries=150, sequenceid=452, filesize=12.0 K 2024-11-12T12:24:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/d99648a30f5b4b12b9792b03639557a2 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d99648a30f5b4b12b9792b03639557a2 2024-11-12T12:24:41,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d99648a30f5b4b12b9792b03639557a2, entries=150, sequenceid=452, filesize=12.0 K 2024-11-12T12:24:41,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 03699d6939471fdc6521cd2a2d843827 in 113ms, sequenceid=452, compaction requested=true 2024-11-12T12:24:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:41,017 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:41,017 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03699d6939471fdc6521cd2a2d843827:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:41,018 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:41,018 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:41,018 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/B is initiating minor compaction (all files) 2024-11-12T12:24:41,018 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/A is initiating minor compaction (all files) 2024-11-12T12:24:41,018 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/B in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,018 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/A in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,018 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/c9ad1693de524288a28f16cdc7a9d032, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/360a964f5047497ca79c500525981289, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6fc7075bc49643058135aac0f94cc23b] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=37.0 K 2024-11-12T12:24:41,018 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/065c088cff6b4ad9865cf34cb82315e3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/dd5982a0f2784dc3814cbcedf2ed6201, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/fddfd29f5e7744aba986c93576c9a272] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=39.4 K 2024-11-12T12:24:41,019 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ad1693de524288a28f16cdc7a9d032, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731414279659 2024-11-12T12:24:41,019 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 065c088cff6b4ad9865cf34cb82315e3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731414279659 2024-11-12T12:24:41,019 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 360a964f5047497ca79c500525981289, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731414279996 2024-11-12T12:24:41,019 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd5982a0f2784dc3814cbcedf2ed6201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731414279996 2024-11-12T12:24:41,020 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fc7075bc49643058135aac0f94cc23b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731414280744 2024-11-12T12:24:41,020 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting fddfd29f5e7744aba986c93576c9a272, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731414280744 2024-11-12T12:24:41,029 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#A#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:41,029 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/3c8bc9bd52f24ee38293c1bc48231439 is 50, key is test_row_0/A:col10/1731414280780/Put/seqid=0 2024-11-12T12:24:41,034 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:41,035 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/cb6080e09b274fbdab870a89d04a1995 is 50, key is test_row_0/B:col10/1731414280780/Put/seqid=0 2024-11-12T12:24:41,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741933_1109 (size=13391) 2024-11-12T12:24:41,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741934_1110 (size=13391) 2024-11-12T12:24:41,048 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/3c8bc9bd52f24ee38293c1bc48231439 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/3c8bc9bd52f24ee38293c1bc48231439 2024-11-12T12:24:41,057 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/cb6080e09b274fbdab870a89d04a1995 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/cb6080e09b274fbdab870a89d04a1995 2024-11-12T12:24:41,059 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/A of 03699d6939471fdc6521cd2a2d843827 into 3c8bc9bd52f24ee38293c1bc48231439(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:41,059 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:41,059 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/A, priority=13, startTime=1731414281017; duration=0sec 2024-11-12T12:24:41,059 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:41,059 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:A 2024-11-12T12:24:41,060 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:41,062 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:41,062 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 03699d6939471fdc6521cd2a2d843827/C is initiating minor compaction (all files) 2024-11-12T12:24:41,062 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03699d6939471fdc6521cd2a2d843827/C in TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,062 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fd7a03ba4dd049b1b75ab8d2728867aa, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c18d49dccfc145d7aa269115e4028919, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d99648a30f5b4b12b9792b03639557a2] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp, totalSize=37.0 K 2024-11-12T12:24:41,063 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd7a03ba4dd049b1b75ab8d2728867aa, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731414279659 2024-11-12T12:24:41,063 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting c18d49dccfc145d7aa269115e4028919, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731414279996 2024-11-12T12:24:41,063 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting d99648a30f5b4b12b9792b03639557a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731414280744 2024-11-12T12:24:41,065 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/B of 03699d6939471fdc6521cd2a2d843827 into cb6080e09b274fbdab870a89d04a1995(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:41,066 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:41,066 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/B, priority=13, startTime=1731414281017; duration=0sec 2024-11-12T12:24:41,066 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:41,066 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:B 2024-11-12T12:24:41,071 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03699d6939471fdc6521cd2a2d843827#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:41,072 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/696ac7a6900145f694061a7e8f67e10e is 50, key is test_row_0/C:col10/1731414280780/Put/seqid=0 2024-11-12T12:24:41,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741935_1111 (size=13391) 2024-11-12T12:24:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:41,082 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:41,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-12T12:24:41,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/efd05ed0519f40e7aa90ea568c5513ee is 50, key is test_row_0/A:col10/1731414281081/Put/seqid=0 2024-11-12T12:24:41,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741936_1112 (size=14741) 2024-11-12T12:24:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T12:24:41,101 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:41,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-12T12:24:41,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T12:24:41,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:41,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:41,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414341157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T12:24:41,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-12T12:24:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414341270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:41,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-12T12:24:41,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:41,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T12:24:41,481 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/696ac7a6900145f694061a7e8f67e10e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/696ac7a6900145f694061a7e8f67e10e 2024-11-12T12:24:41,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414341481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:41,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:41,487 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03699d6939471fdc6521cd2a2d843827/C of 03699d6939471fdc6521cd2a2d843827 into 696ac7a6900145f694061a7e8f67e10e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:41,487 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:41,487 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., storeName=03699d6939471fdc6521cd2a2d843827/C, priority=13, startTime=1731414281017; duration=0sec 2024-11-12T12:24:41,487 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:41,487 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03699d6939471fdc6521cd2a2d843827:C 2024-11-12T12:24:41,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/efd05ed0519f40e7aa90ea568c5513ee 2024-11-12T12:24:41,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/e81cf296bc834adf9f4c95bd01e4417a is 50, key is test_row_0/B:col10/1731414281081/Put/seqid=0 2024-11-12T12:24:41,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741937_1113 (size=12301) 2024-11-12T12:24:41,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-12T12:24:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-12T12:24:41,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T12:24:41,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414341791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:41,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-12T12:24:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:41,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:41,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/e81cf296bc834adf9f4c95bd01e4417a 2024-11-12T12:24:41,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/6c1e6b5b0efd4da0a883d46d23c6f397 is 50, key is test_row_0/C:col10/1731414281081/Put/seqid=0 2024-11-12T12:24:41,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741938_1114 (size=12301) 2024-11-12T12:24:42,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-12T12:24:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:42,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:42,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731414342151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:42,153 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 03699d6939471fdc6521cd2a2d843827 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:24:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42982 deadline: 1731414342161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:42,164 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 03699d6939471fdc6521cd2a2d843827 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:24:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:42,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43010 deadline: 1731414342171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:42,174 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in 03699d6939471fdc6521cd2a2d843827 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:24:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:42,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-12T12:24:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:42,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:42,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42978 deadline: 1731414342181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:42,182 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 03699d6939471fdc6521cd2a2d843827 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:24:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T12:24:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42996 deadline: 1731414342301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:24:42,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-12T12:24:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:42,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:42,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/6c1e6b5b0efd4da0a883d46d23c6f397 2024-11-12T12:24:42,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/efd05ed0519f40e7aa90ea568c5513ee as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/efd05ed0519f40e7aa90ea568c5513ee 2024-11-12T12:24:42,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/efd05ed0519f40e7aa90ea568c5513ee, entries=200, sequenceid=471, filesize=14.4 K 2024-11-12T12:24:42,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/e81cf296bc834adf9f4c95bd01e4417a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/e81cf296bc834adf9f4c95bd01e4417a 2024-11-12T12:24:42,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/e81cf296bc834adf9f4c95bd01e4417a, entries=150, sequenceid=471, filesize=12.0 K 2024-11-12T12:24:42,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/6c1e6b5b0efd4da0a883d46d23c6f397 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/6c1e6b5b0efd4da0a883d46d23c6f397 2024-11-12T12:24:42,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/6c1e6b5b0efd4da0a883d46d23c6f397, entries=150, sequenceid=471, filesize=12.0 K 2024-11-12T12:24:42,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 03699d6939471fdc6521cd2a2d843827 in 1292ms, sequenceid=471, compaction requested=false 2024-11-12T12:24:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:42,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-12T12:24:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:42,488 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-12T12:24:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/005e5682ddfd4715a047d4e0f13b03d1 is 50, key is test_row_0/A:col10/1731414281151/Put/seqid=0 2024-11-12T12:24:42,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741939_1115 (size=12301) 2024-11-12T12:24:42,900 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/005e5682ddfd4715a047d4e0f13b03d1 2024-11-12T12:24:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/f35132c687d7478d8c214059632216b5 is 50, key is test_row_0/B:col10/1731414281151/Put/seqid=0 2024-11-12T12:24:42,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741940_1116 (size=12301) 2024-11-12T12:24:43,101 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-12T12:24:43,102 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:24:43,103 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-12T12:24:43,103 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:24:43,103 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-12T12:24:43,103 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:24:43,105 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-12T12:24:43,105 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:24:43,106 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,106 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,106 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,106 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,107 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,107 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,107 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:43,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:43,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:43,107 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T12:24:43,312 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/f35132c687d7478d8c214059632216b5 2024-11-12T12:24:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:43,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. as already flushing 2024-11-12T12:24:43,324 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-12T12:24:43,324 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:24:43,324 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,324 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:43,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:43,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/913d6f15073946cbaace7f91b8bd4fcb is 50, key is test_row_0/C:col10/1731414281151/Put/seqid=0 2024-11-12T12:24:43,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741941_1117 (size=12301) 2024-11-12T12:24:43,730 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/913d6f15073946cbaace7f91b8bd4fcb 2024-11-12T12:24:43,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/005e5682ddfd4715a047d4e0f13b03d1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/005e5682ddfd4715a047d4e0f13b03d1 2024-11-12T12:24:43,745 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/005e5682ddfd4715a047d4e0f13b03d1, entries=150, sequenceid=492, filesize=12.0 K 2024-11-12T12:24:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/f35132c687d7478d8c214059632216b5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f35132c687d7478d8c214059632216b5 2024-11-12T12:24:43,751 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f35132c687d7478d8c214059632216b5, entries=150, sequenceid=492, filesize=12.0 K 2024-11-12T12:24:43,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/913d6f15073946cbaace7f91b8bd4fcb as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/913d6f15073946cbaace7f91b8bd4fcb 2024-11-12T12:24:43,757 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/913d6f15073946cbaace7f91b8bd4fcb, entries=150, sequenceid=492, filesize=12.0 K 2024-11-12T12:24:43,758 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=6.71 KB/6870 for 03699d6939471fdc6521cd2a2d843827 in 1271ms, sequenceid=492, compaction requested=true 2024-11-12T12:24:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 03699d6939471fdc6521cd2a2d843827: 2024-11-12T12:24:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-12T12:24:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-12T12:24:43,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-12T12:24:43,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6550 sec 2024-11-12T12:24:43,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 2.6590 sec 2024-11-12T12:24:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T12:24:45,262 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:46,045 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T12:24:46,176 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-12T12:24:46,176 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:24:46,177 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,177 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:46,193 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-12T12:24:46,193 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:24:46,193 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,193 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:46,212 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-12T12:24:46,212 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:24:46,212 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,212 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:46,222 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-12T12:24:46,222 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:24:46,222 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,222 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3441 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3424 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1550 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4644 rows 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1561 2024-11-12T12:24:46,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4674 rows 2024-11-12T12:24:46,224 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T12:24:46,224 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T12:24:46,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:46,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:24:46,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T12:24:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T12:24:46,233 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414286233"}]},"ts":"1731414286233"} 2024-11-12T12:24:46,235 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T12:24:46,235 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T12:24:46,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T12:24:46,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03699d6939471fdc6521cd2a2d843827, UNASSIGN}] 2024-11-12T12:24:46,242 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03699d6939471fdc6521cd2a2d843827, UNASSIGN 2024-11-12T12:24:46,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=03699d6939471fdc6521cd2a2d843827, regionState=CLOSING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:46,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=03699d6939471fdc6521cd2a2d843827, UNASSIGN because future has completed 2024-11-12T12:24:46,247 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T12:24:46,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:24:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T12:24:46,408 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:46,408 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T12:24:46,410 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 03699d6939471fdc6521cd2a2d843827, disabling compactions & flushes 2024-11-12T12:24:46,410 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:46,410 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:46,410 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. after waiting 0 ms 2024-11-12T12:24:46,410 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:46,410 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 03699d6939471fdc6521cd2a2d843827 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-12T12:24:46,415 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/faa48f5d47e742268585dd7b15c9df30 is 50, key is test_row_1/A:col10/1731414286221/Put/seqid=0 2024-11-12T12:24:46,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741942_1118 (size=9857) 2024-11-12T12:24:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T12:24:46,821 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/faa48f5d47e742268585dd7b15c9df30 2024-11-12T12:24:46,835 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/e64c101afdb54b9a8d5fc0d63cd418e5 is 50, key is test_row_1/B:col10/1731414286221/Put/seqid=0 2024-11-12T12:24:46,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741943_1119 (size=9857) 2024-11-12T12:24:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T12:24:47,241 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/e64c101afdb54b9a8d5fc0d63cd418e5 2024-11-12T12:24:47,257 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/ac9666f13f4a4497a15f94eae3d926b5 is 50, key is test_row_1/C:col10/1731414286221/Put/seqid=0 2024-11-12T12:24:47,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741944_1120 (size=9857) 2024-11-12T12:24:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T12:24:47,662 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/ac9666f13f4a4497a15f94eae3d926b5 2024-11-12T12:24:47,673 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/A/faa48f5d47e742268585dd7b15c9df30 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/faa48f5d47e742268585dd7b15c9df30 2024-11-12T12:24:47,678 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/faa48f5d47e742268585dd7b15c9df30, entries=100, sequenceid=500, filesize=9.6 K 2024-11-12T12:24:47,679 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/B/e64c101afdb54b9a8d5fc0d63cd418e5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/e64c101afdb54b9a8d5fc0d63cd418e5 2024-11-12T12:24:47,683 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/e64c101afdb54b9a8d5fc0d63cd418e5, entries=100, sequenceid=500, filesize=9.6 K 2024-11-12T12:24:47,684 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/.tmp/C/ac9666f13f4a4497a15f94eae3d926b5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9666f13f4a4497a15f94eae3d926b5 2024-11-12T12:24:47,689 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9666f13f4a4497a15f94eae3d926b5, entries=100, sequenceid=500, filesize=9.6 K 2024-11-12T12:24:47,690 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 03699d6939471fdc6521cd2a2d843827 in 1280ms, sequenceid=500, compaction requested=true 2024-11-12T12:24:47,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/facbb841fdfa45899924a1f220078511, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/1fa6ba37d19d4ac09e7eca4c74f96110, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c89d4a2eed84455ba1f4d3a7d55e30a0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/018d18587a4040fea825f6fe8a103e4a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5e4b5c82bec040daa66179278853d6f9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/7f0440db09574651ab79db9c06da4f91, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/a81cbc13c9e44616bad14940bcc31c03, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/75dc256688744c3c831dd22f116dcf7a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5adc760820624aa29a55bb852a28e952, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bddb7a77ae9f44d1a97623dc9e8db8fd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/da5700a0da8541ba8554665c553af4e5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/6cf35ce6e63c4cb1a399f4a2b3ce85ca, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/3171a59752e1444d9feed738a7d6c029, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c5636f2281e843eb9fe2232ee0823bcf, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/500e029ab5224e5d9e8d07824eb9703d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c25b567e603d452ba9bd4f5d3dbd563c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5b33cc625a454b5e8002dd12d7c51733, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/07f50c80f67049f3a8207daa10431574, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/14181751edb4485f9309380ab515e217, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f59ce7ef14a346f5933c55a54b91ad26, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/38e3648d50b443bc903a28804e34566d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/cc6d7994f5ec4d08a49d554b26508e7a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/ec428c42f20b4145ac04be259bd94fc1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bf325a5c06004ecda0df36e8795c2479, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4ddf4a4c8d9d49259090171837ca7305, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/aeee2ae450694a5091ccac132401fb6a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4144609f64664449962fb4aedfdc8b5e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/9beeca6134d9434097e2282f17226699, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f7c7110adedd42e384b923fe0b00765c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/065c088cff6b4ad9865cf34cb82315e3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/dd5982a0f2784dc3814cbcedf2ed6201, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/fddfd29f5e7744aba986c93576c9a272] to archive 2024-11-12T12:24:47,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:24:47,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/facbb841fdfa45899924a1f220078511 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/facbb841fdfa45899924a1f220078511 2024-11-12T12:24:47,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/1fa6ba37d19d4ac09e7eca4c74f96110 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/1fa6ba37d19d4ac09e7eca4c74f96110 2024-11-12T12:24:47,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c89d4a2eed84455ba1f4d3a7d55e30a0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c89d4a2eed84455ba1f4d3a7d55e30a0 2024-11-12T12:24:47,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/018d18587a4040fea825f6fe8a103e4a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/018d18587a4040fea825f6fe8a103e4a 2024-11-12T12:24:47,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5e4b5c82bec040daa66179278853d6f9 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5e4b5c82bec040daa66179278853d6f9 2024-11-12T12:24:47,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/7f0440db09574651ab79db9c06da4f91 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/7f0440db09574651ab79db9c06da4f91 2024-11-12T12:24:47,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/a81cbc13c9e44616bad14940bcc31c03 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/a81cbc13c9e44616bad14940bcc31c03 2024-11-12T12:24:47,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/75dc256688744c3c831dd22f116dcf7a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/75dc256688744c3c831dd22f116dcf7a 2024-11-12T12:24:47,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5adc760820624aa29a55bb852a28e952 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5adc760820624aa29a55bb852a28e952 2024-11-12T12:24:47,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bddb7a77ae9f44d1a97623dc9e8db8fd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bddb7a77ae9f44d1a97623dc9e8db8fd 2024-11-12T12:24:47,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/da5700a0da8541ba8554665c553af4e5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/da5700a0da8541ba8554665c553af4e5 2024-11-12T12:24:47,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/6cf35ce6e63c4cb1a399f4a2b3ce85ca to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/6cf35ce6e63c4cb1a399f4a2b3ce85ca 2024-11-12T12:24:47,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/3171a59752e1444d9feed738a7d6c029 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/3171a59752e1444d9feed738a7d6c029 2024-11-12T12:24:47,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c5636f2281e843eb9fe2232ee0823bcf to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c5636f2281e843eb9fe2232ee0823bcf 2024-11-12T12:24:47,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/500e029ab5224e5d9e8d07824eb9703d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/500e029ab5224e5d9e8d07824eb9703d 2024-11-12T12:24:47,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c25b567e603d452ba9bd4f5d3dbd563c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/c25b567e603d452ba9bd4f5d3dbd563c 2024-11-12T12:24:47,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5b33cc625a454b5e8002dd12d7c51733 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/5b33cc625a454b5e8002dd12d7c51733 2024-11-12T12:24:47,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/07f50c80f67049f3a8207daa10431574 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/07f50c80f67049f3a8207daa10431574 2024-11-12T12:24:47,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/14181751edb4485f9309380ab515e217 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/14181751edb4485f9309380ab515e217 2024-11-12T12:24:47,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f59ce7ef14a346f5933c55a54b91ad26 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f59ce7ef14a346f5933c55a54b91ad26 2024-11-12T12:24:47,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/38e3648d50b443bc903a28804e34566d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/38e3648d50b443bc903a28804e34566d 2024-11-12T12:24:47,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/cc6d7994f5ec4d08a49d554b26508e7a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/cc6d7994f5ec4d08a49d554b26508e7a 2024-11-12T12:24:47,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/ec428c42f20b4145ac04be259bd94fc1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/ec428c42f20b4145ac04be259bd94fc1 2024-11-12T12:24:47,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bf325a5c06004ecda0df36e8795c2479 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/bf325a5c06004ecda0df36e8795c2479 2024-11-12T12:24:47,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4ddf4a4c8d9d49259090171837ca7305 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4ddf4a4c8d9d49259090171837ca7305 2024-11-12T12:24:47,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/aeee2ae450694a5091ccac132401fb6a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/aeee2ae450694a5091ccac132401fb6a 2024-11-12T12:24:47,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4144609f64664449962fb4aedfdc8b5e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/4144609f64664449962fb4aedfdc8b5e 2024-11-12T12:24:47,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/9beeca6134d9434097e2282f17226699 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/9beeca6134d9434097e2282f17226699 2024-11-12T12:24:47,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f7c7110adedd42e384b923fe0b00765c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/f7c7110adedd42e384b923fe0b00765c 2024-11-12T12:24:47,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/065c088cff6b4ad9865cf34cb82315e3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/065c088cff6b4ad9865cf34cb82315e3 2024-11-12T12:24:47,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/dd5982a0f2784dc3814cbcedf2ed6201 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/dd5982a0f2784dc3814cbcedf2ed6201 2024-11-12T12:24:47,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/fddfd29f5e7744aba986c93576c9a272 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/fddfd29f5e7744aba986c93576c9a272 2024-11-12T12:24:47,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7cbc55b6f9d94b3e9f48f3235edabbbf, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/00734457a1d5401b8e92b5ce37bef9ba, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/779306dd4e3249ad96bba33b44142d91, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7ff799b9c18b47beaf711db369f159e5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f223111409b646c0b0b384b53202a046, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/44549df3b14243e78b1bf8e05a858506, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/33010ec21e5f4f0fb806c9bb9693c204, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/a08f07145d5440eba237833a50c242c3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/226600a973a04b7a8a0ce7fb6dd9e502, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2768685ab11d41fbb1c92f2b55b02070, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/9174394c5ae94b8f93b31eaf665b5f25, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2541ba74f9734060a8f04641509cc9bd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d43edf058d264b51ab2a70ea057ee147, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/54f062f85d3e4bc2a62f7d81136c38fd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/0f0ff4d60f744da189c6db26da53b472, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d0f9237e8e0c4a048e21a3cbcb85b7ec, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/92ae11d8bd05409eaf2221de13d00709, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/28ad977d815a451a96aa8badcae0dcdd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/20ca4628f1bc4db7b655c211f1e3dde1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/46fcb01708374dc1b51ee6a609bb7471, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6f402edecb2144a1ba1bcc8122e8348f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/885c2016f16a4dbdbfe595a03f893693, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b791dc048a39415d8ec15b4e781f5264, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b239d9395c054be3aec813e0fa8a2b05, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/38561494d13342839c5d9060a43762e0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/614fe388f81741a6930c920a1ebf4b6e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/1e3d01ad51e0409f8afedd6c65e1532f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2efeb23abc50426ab940ddd8be90ba01, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/c9ad1693de524288a28f16cdc7a9d032, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/19bc9744295a4787ad287da7c471ce8b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/360a964f5047497ca79c500525981289, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6fc7075bc49643058135aac0f94cc23b] to archive 2024-11-12T12:24:47,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:24:47,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7cbc55b6f9d94b3e9f48f3235edabbbf to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7cbc55b6f9d94b3e9f48f3235edabbbf 2024-11-12T12:24:47,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/00734457a1d5401b8e92b5ce37bef9ba to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/00734457a1d5401b8e92b5ce37bef9ba 2024-11-12T12:24:47,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/779306dd4e3249ad96bba33b44142d91 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/779306dd4e3249ad96bba33b44142d91 2024-11-12T12:24:47,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7ff799b9c18b47beaf711db369f159e5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/7ff799b9c18b47beaf711db369f159e5 2024-11-12T12:24:47,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f223111409b646c0b0b384b53202a046 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f223111409b646c0b0b384b53202a046 2024-11-12T12:24:47,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/44549df3b14243e78b1bf8e05a858506 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/44549df3b14243e78b1bf8e05a858506 2024-11-12T12:24:47,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/33010ec21e5f4f0fb806c9bb9693c204 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/33010ec21e5f4f0fb806c9bb9693c204 2024-11-12T12:24:47,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/a08f07145d5440eba237833a50c242c3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/a08f07145d5440eba237833a50c242c3 2024-11-12T12:24:47,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/226600a973a04b7a8a0ce7fb6dd9e502 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/226600a973a04b7a8a0ce7fb6dd9e502 2024-11-12T12:24:47,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2768685ab11d41fbb1c92f2b55b02070 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2768685ab11d41fbb1c92f2b55b02070 2024-11-12T12:24:47,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/9174394c5ae94b8f93b31eaf665b5f25 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/9174394c5ae94b8f93b31eaf665b5f25 2024-11-12T12:24:47,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2541ba74f9734060a8f04641509cc9bd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2541ba74f9734060a8f04641509cc9bd 2024-11-12T12:24:47,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d43edf058d264b51ab2a70ea057ee147 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d43edf058d264b51ab2a70ea057ee147 2024-11-12T12:24:47,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/54f062f85d3e4bc2a62f7d81136c38fd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/54f062f85d3e4bc2a62f7d81136c38fd 2024-11-12T12:24:47,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/0f0ff4d60f744da189c6db26da53b472 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/0f0ff4d60f744da189c6db26da53b472 2024-11-12T12:24:47,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d0f9237e8e0c4a048e21a3cbcb85b7ec to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/d0f9237e8e0c4a048e21a3cbcb85b7ec 2024-11-12T12:24:47,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/92ae11d8bd05409eaf2221de13d00709 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/92ae11d8bd05409eaf2221de13d00709 2024-11-12T12:24:47,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/28ad977d815a451a96aa8badcae0dcdd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/28ad977d815a451a96aa8badcae0dcdd 2024-11-12T12:24:47,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/20ca4628f1bc4db7b655c211f1e3dde1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/20ca4628f1bc4db7b655c211f1e3dde1 2024-11-12T12:24:47,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/46fcb01708374dc1b51ee6a609bb7471 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/46fcb01708374dc1b51ee6a609bb7471 2024-11-12T12:24:47,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6f402edecb2144a1ba1bcc8122e8348f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6f402edecb2144a1ba1bcc8122e8348f 2024-11-12T12:24:47,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/885c2016f16a4dbdbfe595a03f893693 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/885c2016f16a4dbdbfe595a03f893693 2024-11-12T12:24:47,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b791dc048a39415d8ec15b4e781f5264 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b791dc048a39415d8ec15b4e781f5264 2024-11-12T12:24:47,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b239d9395c054be3aec813e0fa8a2b05 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/b239d9395c054be3aec813e0fa8a2b05 2024-11-12T12:24:47,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/38561494d13342839c5d9060a43762e0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/38561494d13342839c5d9060a43762e0 2024-11-12T12:24:47,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/614fe388f81741a6930c920a1ebf4b6e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/614fe388f81741a6930c920a1ebf4b6e 2024-11-12T12:24:47,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/1e3d01ad51e0409f8afedd6c65e1532f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/1e3d01ad51e0409f8afedd6c65e1532f 2024-11-12T12:24:47,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2efeb23abc50426ab940ddd8be90ba01 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/2efeb23abc50426ab940ddd8be90ba01 2024-11-12T12:24:47,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/c9ad1693de524288a28f16cdc7a9d032 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/c9ad1693de524288a28f16cdc7a9d032 2024-11-12T12:24:47,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/19bc9744295a4787ad287da7c471ce8b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/19bc9744295a4787ad287da7c471ce8b 2024-11-12T12:24:47,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/360a964f5047497ca79c500525981289 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/360a964f5047497ca79c500525981289 2024-11-12T12:24:47,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6fc7075bc49643058135aac0f94cc23b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/6fc7075bc49643058135aac0f94cc23b 2024-11-12T12:24:47,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/20cbb6b8595c4a3799b6898d97584488, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fbcbcd881b02436db4aa6526b57d7e97, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/027f889137a0492c82ee6a5bd54e96b6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a23931d80db24b2d85e8b339538de841, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a3195be70a744c979a41d20bcee0cb13, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3867eff0c91645c9b2387f7b24020d46, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/bd68dc502dc74a1db66a618fc5e479c7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d7666ae285474aeda6a6983e681cb023, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/8a8f258a2d3d4f439036ba0ae7a127c5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d172e63aa18e4d55aed3e59049456283, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7adaeb1ca3494cddbc94fef4152fc297, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b9edb9c47f184b76929f6d912367eb06, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b5aec0b0c75549b7a2305a87cf15e9b1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/077b1e496725411287e7ab9d4815d335, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/1619b644aba1450a98ae8087cb6fa7d4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c05f7e6b3c784a22b8567fcd80c933a7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3339eda7a14f4156a4fec4f4bb9c0379, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ef7506695e0340fa9d224340aa0d9d75, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/57598079a7e643b9893cd7c77972e94b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a07c48f3b6ec414b8e5bc18ccc06a2ab, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7b837eada3664dfca48632663d56dc44, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/505d981999914fe9917bd7b59484e81b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/540ca5e8ea7a4f5aa21c8dce46b27fb4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/13970f8e64d94c0cba361a352908f3bc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9011a1ebca4bcba763d12cb9f20596, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/baea77712f3f40cfad7767550f5d81b5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/681129b5a5944c438b225205526adfc6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/aec900831d40479dbb5ea181afa2f033, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fd7a03ba4dd049b1b75ab8d2728867aa, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/0ccf0c5d99254876ac0c590a43b37836, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c18d49dccfc145d7aa269115e4028919, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d99648a30f5b4b12b9792b03639557a2] to archive 2024-11-12T12:24:47,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:24:47,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/20cbb6b8595c4a3799b6898d97584488 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/20cbb6b8595c4a3799b6898d97584488 2024-11-12T12:24:47,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fbcbcd881b02436db4aa6526b57d7e97 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fbcbcd881b02436db4aa6526b57d7e97 2024-11-12T12:24:47,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/027f889137a0492c82ee6a5bd54e96b6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/027f889137a0492c82ee6a5bd54e96b6 2024-11-12T12:24:47,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a23931d80db24b2d85e8b339538de841 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a23931d80db24b2d85e8b339538de841 2024-11-12T12:24:47,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a3195be70a744c979a41d20bcee0cb13 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a3195be70a744c979a41d20bcee0cb13 2024-11-12T12:24:47,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3867eff0c91645c9b2387f7b24020d46 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3867eff0c91645c9b2387f7b24020d46 2024-11-12T12:24:47,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/bd68dc502dc74a1db66a618fc5e479c7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/bd68dc502dc74a1db66a618fc5e479c7 2024-11-12T12:24:47,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d7666ae285474aeda6a6983e681cb023 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d7666ae285474aeda6a6983e681cb023 2024-11-12T12:24:47,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/8a8f258a2d3d4f439036ba0ae7a127c5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/8a8f258a2d3d4f439036ba0ae7a127c5 2024-11-12T12:24:47,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d172e63aa18e4d55aed3e59049456283 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d172e63aa18e4d55aed3e59049456283 2024-11-12T12:24:47,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7adaeb1ca3494cddbc94fef4152fc297 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7adaeb1ca3494cddbc94fef4152fc297 2024-11-12T12:24:47,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b9edb9c47f184b76929f6d912367eb06 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b9edb9c47f184b76929f6d912367eb06 2024-11-12T12:24:47,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b5aec0b0c75549b7a2305a87cf15e9b1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/b5aec0b0c75549b7a2305a87cf15e9b1 2024-11-12T12:24:47,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/077b1e496725411287e7ab9d4815d335 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/077b1e496725411287e7ab9d4815d335 2024-11-12T12:24:47,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/1619b644aba1450a98ae8087cb6fa7d4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/1619b644aba1450a98ae8087cb6fa7d4 2024-11-12T12:24:47,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c05f7e6b3c784a22b8567fcd80c933a7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c05f7e6b3c784a22b8567fcd80c933a7 2024-11-12T12:24:47,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3339eda7a14f4156a4fec4f4bb9c0379 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/3339eda7a14f4156a4fec4f4bb9c0379 2024-11-12T12:24:47,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ef7506695e0340fa9d224340aa0d9d75 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ef7506695e0340fa9d224340aa0d9d75 2024-11-12T12:24:47,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/57598079a7e643b9893cd7c77972e94b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/57598079a7e643b9893cd7c77972e94b 2024-11-12T12:24:47,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a07c48f3b6ec414b8e5bc18ccc06a2ab to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/a07c48f3b6ec414b8e5bc18ccc06a2ab 2024-11-12T12:24:47,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7b837eada3664dfca48632663d56dc44 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/7b837eada3664dfca48632663d56dc44 2024-11-12T12:24:47,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/505d981999914fe9917bd7b59484e81b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/505d981999914fe9917bd7b59484e81b 2024-11-12T12:24:47,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/540ca5e8ea7a4f5aa21c8dce46b27fb4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/540ca5e8ea7a4f5aa21c8dce46b27fb4 2024-11-12T12:24:47,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/13970f8e64d94c0cba361a352908f3bc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/13970f8e64d94c0cba361a352908f3bc 2024-11-12T12:24:47,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9011a1ebca4bcba763d12cb9f20596 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9011a1ebca4bcba763d12cb9f20596 2024-11-12T12:24:47,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/baea77712f3f40cfad7767550f5d81b5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/baea77712f3f40cfad7767550f5d81b5 2024-11-12T12:24:47,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/681129b5a5944c438b225205526adfc6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/681129b5a5944c438b225205526adfc6 2024-11-12T12:24:47,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/aec900831d40479dbb5ea181afa2f033 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/aec900831d40479dbb5ea181afa2f033 2024-11-12T12:24:47,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fd7a03ba4dd049b1b75ab8d2728867aa to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/fd7a03ba4dd049b1b75ab8d2728867aa 2024-11-12T12:24:47,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/0ccf0c5d99254876ac0c590a43b37836 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/0ccf0c5d99254876ac0c590a43b37836 2024-11-12T12:24:47,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c18d49dccfc145d7aa269115e4028919 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/c18d49dccfc145d7aa269115e4028919 2024-11-12T12:24:47,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d99648a30f5b4b12b9792b03639557a2 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/d99648a30f5b4b12b9792b03639557a2 2024-11-12T12:24:47,823 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/recovered.edits/503.seqid, newMaxSeqId=503, maxSeqId=1 2024-11-12T12:24:47,824 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. 2024-11-12T12:24:47,824 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 03699d6939471fdc6521cd2a2d843827: Waiting for close lock at 1731414286409Running coprocessor pre-close hooks at 1731414286409Disabling compacts and flushes for region at 1731414286409Disabling writes for close at 1731414286410 (+1 ms)Obtaining lock to block concurrent updates at 1731414286410Preparing flush snapshotting stores in 03699d6939471fdc6521cd2a2d843827 at 1731414286410Finished memstore snapshotting TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731414286411 (+1 ms)Flushing stores of TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827. at 1731414286412 (+1 ms)Flushing 03699d6939471fdc6521cd2a2d843827/A: creating writer at 1731414286412Flushing 03699d6939471fdc6521cd2a2d843827/A: appending metadata at 1731414286415 (+3 ms)Flushing 03699d6939471fdc6521cd2a2d843827/A: closing flushed file at 1731414286415Flushing 03699d6939471fdc6521cd2a2d843827/B: creating writer at 1731414286831 (+416 ms)Flushing 03699d6939471fdc6521cd2a2d843827/B: appending metadata at 1731414286834 (+3 ms)Flushing 03699d6939471fdc6521cd2a2d843827/B: closing flushed file at 1731414286834Flushing 03699d6939471fdc6521cd2a2d843827/C: creating writer at 1731414287252 (+418 ms)Flushing 03699d6939471fdc6521cd2a2d843827/C: appending metadata at 1731414287256 (+4 ms)Flushing 03699d6939471fdc6521cd2a2d843827/C: closing flushed file at 1731414287256Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e356894: reopening flushed file at 1731414287672 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a44d320: reopening flushed file at 1731414287678 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@240d58d9: reopening flushed file at 1731414287683 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 03699d6939471fdc6521cd2a2d843827 in 1280ms, sequenceid=500, compaction requested=true at 1731414287690 (+7 ms)Writing region close event to WAL at 1731414287820 (+130 ms)Running coprocessor post-close hooks at 1731414287824 (+4 ms)Closed at 1731414287824 2024-11-12T12:24:47,827 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:47,827 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=03699d6939471fdc6521cd2a2d843827, regionState=CLOSED 2024-11-12T12:24:47,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:24:47,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-12T12:24:47,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 03699d6939471fdc6521cd2a2d843827, server=4a91c05c96a4,38717,1731414257691 in 1.5830 sec 2024-11-12T12:24:47,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-12T12:24:47,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03699d6939471fdc6521cd2a2d843827, UNASSIGN in 1.5910 sec 2024-11-12T12:24:47,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T12:24:47,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5980 sec 2024-11-12T12:24:47,838 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414287838"}]},"ts":"1731414287838"} 2024-11-12T12:24:47,840 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T12:24:47,840 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T12:24:47,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6120 sec 2024-11-12T12:24:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T12:24:48,383 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:48,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T12:24:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:48,395 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-12T12:24:48,396 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:48,400 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:48,406 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/recovered.edits] 2024-11-12T12:24:48,410 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/005e5682ddfd4715a047d4e0f13b03d1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/005e5682ddfd4715a047d4e0f13b03d1 2024-11-12T12:24:48,412 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/3c8bc9bd52f24ee38293c1bc48231439 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/3c8bc9bd52f24ee38293c1bc48231439 2024-11-12T12:24:48,414 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/efd05ed0519f40e7aa90ea568c5513ee to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/efd05ed0519f40e7aa90ea568c5513ee 2024-11-12T12:24:48,416 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/faa48f5d47e742268585dd7b15c9df30 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/A/faa48f5d47e742268585dd7b15c9df30 2024-11-12T12:24:48,419 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/cb6080e09b274fbdab870a89d04a1995 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/cb6080e09b274fbdab870a89d04a1995 2024-11-12T12:24:48,420 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/e64c101afdb54b9a8d5fc0d63cd418e5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/e64c101afdb54b9a8d5fc0d63cd418e5 2024-11-12T12:24:48,422 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/e81cf296bc834adf9f4c95bd01e4417a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/e81cf296bc834adf9f4c95bd01e4417a 2024-11-12T12:24:48,423 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f35132c687d7478d8c214059632216b5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/B/f35132c687d7478d8c214059632216b5 2024-11-12T12:24:48,426 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/696ac7a6900145f694061a7e8f67e10e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/696ac7a6900145f694061a7e8f67e10e 2024-11-12T12:24:48,427 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/6c1e6b5b0efd4da0a883d46d23c6f397 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/6c1e6b5b0efd4da0a883d46d23c6f397 2024-11-12T12:24:48,429 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/913d6f15073946cbaace7f91b8bd4fcb to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/913d6f15073946cbaace7f91b8bd4fcb 2024-11-12T12:24:48,430 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9666f13f4a4497a15f94eae3d926b5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/C/ac9666f13f4a4497a15f94eae3d926b5 2024-11-12T12:24:48,433 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/recovered.edits/503.seqid to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827/recovered.edits/503.seqid 2024-11-12T12:24:48,434 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/03699d6939471fdc6521cd2a2d843827 2024-11-12T12:24:48,434 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T12:24:48,441 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:48,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-12T12:24:48,451 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T12:24:48,482 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T12:24:48,483 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:48,483 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T12:24:48,484 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731414288483"}]},"ts":"9223372036854775807"} 2024-11-12T12:24:48,489 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T12:24:48,489 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 03699d6939471fdc6521cd2a2d843827, NAME => 'TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T12:24:48,489 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T12:24:48,490 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731414288489"}]},"ts":"9223372036854775807"} 2024-11-12T12:24:48,493 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T12:24:48,494 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:48,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 103 msec 2024-11-12T12:24:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-12T12:24:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T12:24:48,503 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:48,513 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;4a91c05c96a4:38717-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1936992572_22 at /127.0.0.1:42852 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=496 (was 365) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7218 (was 8482) 2024-11-12T12:24:48,521 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=496, ProcessCount=11, AvailableMemoryMB=7217 2024-11-12T12:24:48,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T12:24:48,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:24:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:48,526 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T12:24:48,526 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:48,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-12T12:24:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T12:24:48,527 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T12:24:48,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741945_1121 (size=959) 2024-11-12T12:24:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T12:24:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T12:24:48,939 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a 2024-11-12T12:24:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741946_1122 (size=53) 2024-11-12T12:24:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T12:24:49,356 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:24:49,356 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cfcbe6d6503fe59cbb8c8cb9085531ad, disabling compactions & flushes 2024-11-12T12:24:49,357 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:49,357 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:49,357 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. after waiting 0 ms 2024-11-12T12:24:49,357 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:49,357 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:49,357 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cfcbe6d6503fe59cbb8c8cb9085531ad: Waiting for close lock at 1731414289356Disabling compacts and flushes for region at 1731414289356Disabling writes for close at 1731414289357 (+1 ms)Writing region close event to WAL at 1731414289357Closed at 1731414289357 2024-11-12T12:24:49,361 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T12:24:49,362 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731414289361"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731414289361"}]},"ts":"1731414289361"} 2024-11-12T12:24:49,368 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T12:24:49,369 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T12:24:49,369 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414289369"}]},"ts":"1731414289369"} 2024-11-12T12:24:49,370 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T12:24:49,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, ASSIGN}] 2024-11-12T12:24:49,372 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, ASSIGN 2024-11-12T12:24:49,372 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, ASSIGN; state=OFFLINE, location=4a91c05c96a4,38717,1731414257691; forceNewPlan=false, retain=false 2024-11-12T12:24:49,524 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=cfcbe6d6503fe59cbb8c8cb9085531ad, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:49,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, ASSIGN because future has completed 2024-11-12T12:24:49,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:24:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T12:24:49,692 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:49,692 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:24:49,693 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,694 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:24:49,694 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,694 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,697 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,699 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcbe6d6503fe59cbb8c8cb9085531ad columnFamilyName A 2024-11-12T12:24:49,700 DEBUG [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:49,700 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(327): Store=cfcbe6d6503fe59cbb8c8cb9085531ad/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:49,700 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,702 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcbe6d6503fe59cbb8c8cb9085531ad columnFamilyName B 2024-11-12T12:24:49,702 DEBUG [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:49,702 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(327): Store=cfcbe6d6503fe59cbb8c8cb9085531ad/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:49,702 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,704 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcbe6d6503fe59cbb8c8cb9085531ad columnFamilyName C 2024-11-12T12:24:49,704 DEBUG [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:49,704 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(327): Store=cfcbe6d6503fe59cbb8c8cb9085531ad/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:49,704 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,705 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,705 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,707 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,707 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,707 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:24:49,709 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,711 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T12:24:49,712 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened cfcbe6d6503fe59cbb8c8cb9085531ad; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74140555, jitterRate=0.10478036105632782}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:24:49,712 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:49,713 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for cfcbe6d6503fe59cbb8c8cb9085531ad: Running coprocessor pre-open hook at 1731414289695Writing region info on filesystem at 1731414289695Initializing all the Stores at 1731414289696 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414289696Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414289697 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414289697Cleaning up temporary data from old regions at 1731414289707 (+10 ms)Running coprocessor post-open hooks at 1731414289712 (+5 ms)Region opened successfully at 1731414289713 (+1 ms) 2024-11-12T12:24:49,714 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., pid=36, masterSystemTime=1731414289684 2024-11-12T12:24:49,716 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:49,716 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:49,717 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=cfcbe6d6503fe59cbb8c8cb9085531ad, regionState=OPEN, openSeqNum=2, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:49,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:24:49,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-12T12:24:49,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 in 190 msec 2024-11-12T12:24:49,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-12T12:24:49,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, ASSIGN in 351 msec 2024-11-12T12:24:49,724 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T12:24:49,724 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414289724"}]},"ts":"1731414289724"} 2024-11-12T12:24:49,726 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T12:24:49,726 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T12:24:49,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2030 sec 2024-11-12T12:24:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T12:24:50,671 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:50,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:50,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:50,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:50,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2871e361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:50,677 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7e2b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:50,679 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:50,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:50,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T12:24:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a91c05c96a4,41939,1731414256942 2024-11-12T12:24:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@557415e3 2024-11-12T12:24:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T12:24:50,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T12:24:50,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T12:24:50,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:24:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T12:24:50,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741947_1123 (size=995) 2024-11-12T12:24:51,109 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-12T12:24:51,110 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-12T12:24:51,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T12:24:51,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, REOPEN/MOVE}] 2024-11-12T12:24:51,123 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, REOPEN/MOVE 2024-11-12T12:24:51,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=cfcbe6d6503fe59cbb8c8cb9085531ad, regionState=CLOSING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:51,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, REOPEN/MOVE because future has completed 2024-11-12T12:24:51,126 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T12:24:51,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:24:51,279 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,279 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T12:24:51,279 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing cfcbe6d6503fe59cbb8c8cb9085531ad, disabling compactions & flushes 2024-11-12T12:24:51,279 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:51,279 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:51,279 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. after waiting 0 ms 2024-11-12T12:24:51,279 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:51,283 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T12:24:51,284 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:51,284 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for cfcbe6d6503fe59cbb8c8cb9085531ad: Waiting for close lock at 1731414291279Running coprocessor pre-close hooks at 1731414291279Disabling compacts and flushes for region at 1731414291279Disabling writes for close at 1731414291279Writing region close event to WAL at 1731414291280 (+1 ms)Running coprocessor post-close hooks at 1731414291284 (+4 ms)Closed at 1731414291284 2024-11-12T12:24:51,284 WARN [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: cfcbe6d6503fe59cbb8c8cb9085531ad to self. 2024-11-12T12:24:51,287 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,287 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=cfcbe6d6503fe59cbb8c8cb9085531ad, regionState=CLOSED 2024-11-12T12:24:51,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:24:51,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-12T12:24:51,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 in 165 msec 2024-11-12T12:24:51,294 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, REOPEN/MOVE; state=CLOSED, location=4a91c05c96a4,38717,1731414257691; forceNewPlan=false, retain=true 2024-11-12T12:24:51,445 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=cfcbe6d6503fe59cbb8c8cb9085531ad, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:51,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, REOPEN/MOVE because future has completed 2024-11-12T12:24:51,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:24:51,602 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:51,602 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:24:51,603 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,603 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:24:51,603 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,603 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,605 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,612 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcbe6d6503fe59cbb8c8cb9085531ad columnFamilyName A 2024-11-12T12:24:51,614 DEBUG [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:51,615 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(327): Store=cfcbe6d6503fe59cbb8c8cb9085531ad/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:51,616 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,617 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcbe6d6503fe59cbb8c8cb9085531ad columnFamilyName B 2024-11-12T12:24:51,617 DEBUG [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:51,618 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(327): Store=cfcbe6d6503fe59cbb8c8cb9085531ad/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:51,618 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,619 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcbe6d6503fe59cbb8c8cb9085531ad columnFamilyName C 2024-11-12T12:24:51,619 DEBUG [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:51,620 INFO [StoreOpener-cfcbe6d6503fe59cbb8c8cb9085531ad-1 {}] regionserver.HStore(327): Store=cfcbe6d6503fe59cbb8c8cb9085531ad/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:24:51,620 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,621 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,622 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,623 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,623 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,624 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:24:51,626 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,627 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened cfcbe6d6503fe59cbb8c8cb9085531ad; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73571643, jitterRate=0.09630291163921356}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:24:51,627 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,629 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for cfcbe6d6503fe59cbb8c8cb9085531ad: Running coprocessor pre-open hook at 1731414291603Writing region info on filesystem at 1731414291603Initializing all the Stores at 1731414291604 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414291604Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414291604Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414291604Cleaning up temporary data from old regions at 1731414291623 (+19 ms)Running coprocessor post-open hooks at 1731414291627 (+4 ms)Region opened successfully at 1731414291629 (+2 ms) 2024-11-12T12:24:51,630 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., pid=41, masterSystemTime=1731414291599 2024-11-12T12:24:51,633 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:51,634 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=cfcbe6d6503fe59cbb8c8cb9085531ad, regionState=OPEN, openSeqNum=5, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:51,634 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:51,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:24:51,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-12T12:24:51,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 in 190 msec 2024-11-12T12:24:51,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-12T12:24:51,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, REOPEN/MOVE in 518 msec 2024-11-12T12:24:51,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-12T12:24:51,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-11-12T12:24:51,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 964 msec 2024-11-12T12:24:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-12T12:24:51,662 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:51,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:51,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:51,664 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30653dd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:51,666 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5b1922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:51,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:51,673 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d098d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:51,676 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e52f953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:51,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:51,680 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f1a5dab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:51,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb6d812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:51,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:51,690 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@157c4d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:51,693 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10320ded, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:51,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:51,696 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c4a45a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:51,700 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ad6c1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:51,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:51,707 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391b6bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:51,709 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1468316a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:51,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:51,720 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6054ef04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:51,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:51,723 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:51,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af31bce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:51,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:51,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fb2281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:51,729 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:24:51,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:24:51,733 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:24:51,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:24:51,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:24:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5706f074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:24:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:24:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:24:51,738 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:24:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2ecc06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:24:51,742 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:51,743 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:51,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:51,745 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:51,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:24:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:24:51,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:24:51,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:51,757 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-12T12:24:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T12:24:51,759 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:51,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:51,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:51,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:24:51,771 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:51,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:24:51,778 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:51,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:24:51,792 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:51,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:51,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:24:51,798 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:51,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:51,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:51,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:24:51,817 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:51,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:51,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:51,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:24:51,828 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:24:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:24:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:24:51,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:24:51,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:24:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T12:24:51,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111269124b2726f04932bcc19a2f461b16f5_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414291763/Put/seqid=0 2024-11-12T12:24:51,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741948_1124 (size=14594) 2024-11-12T12:24:51,900 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:51,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111269124b2726f04932bcc19a2f461b16f5_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111269124b2726f04932bcc19a2f461b16f5_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:51,908 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/b2b493879858450c901370a90f513902, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:51,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/b2b493879858450c901370a90f513902 is 175, key is test_row_0/A:col10/1731414291763/Put/seqid=0 2024-11-12T12:24:51,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T12:24:51,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:51,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:51,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:51,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:51,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414351927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:51,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414351929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:51,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414351933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:51,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414351936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:51,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414351936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:51,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741949_1125 (size=39549) 2024-11-12T12:24:52,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414352041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414352043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414352052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414352052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T12:24:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T12:24:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414352052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T12:24:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414352252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414352252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414352271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414352272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414352282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,365 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/b2b493879858450c901370a90f513902 2024-11-12T12:24:52,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T12:24:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T12:24:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/33fec82b8bdc4a968b497313b1bc7697 is 50, key is test_row_0/B:col10/1731414291763/Put/seqid=0 2024-11-12T12:24:52,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741950_1126 (size=12001) 2024-11-12T12:24:52,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/33fec82b8bdc4a968b497313b1bc7697 2024-11-12T12:24:52,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/50e2a1ef938841978f4347aa7ead70d6 is 50, key is test_row_0/C:col10/1731414291763/Put/seqid=0 2024-11-12T12:24:52,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741951_1127 (size=12001) 2024-11-12T12:24:52,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T12:24:52,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:52,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414352571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414352582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414352582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414352582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414352592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:52,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T12:24:52,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:52,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T12:24:52,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:52,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:52,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T12:24:52,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/50e2a1ef938841978f4347aa7ead70d6 2024-11-12T12:24:52,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/b2b493879858450c901370a90f513902 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/b2b493879858450c901370a90f513902 2024-11-12T12:24:52,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/b2b493879858450c901370a90f513902, entries=200, sequenceid=16, filesize=38.6 K 2024-11-12T12:24:52,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/33fec82b8bdc4a968b497313b1bc7697 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/33fec82b8bdc4a968b497313b1bc7697 2024-11-12T12:24:52,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/33fec82b8bdc4a968b497313b1bc7697, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T12:24:52,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/50e2a1ef938841978f4347aa7ead70d6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/50e2a1ef938841978f4347aa7ead70d6 2024-11-12T12:24:52,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/50e2a1ef938841978f4347aa7ead70d6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T12:24:52,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1188ms, sequenceid=16, compaction requested=false 2024-11-12T12:24:52,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:53,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T12:24:53,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:53,021 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:24:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bed0c27904354ea994c3cba45d381849_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414291933/Put/seqid=0 2024-11-12T12:24:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:53,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:53,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741952_1128 (size=12154) 2024-11-12T12:24:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:53,114 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bed0c27904354ea994c3cba45d381849_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bed0c27904354ea994c3cba45d381849_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/009728bf53864216b9856d368c97326c, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/009728bf53864216b9856d368c97326c is 175, key is test_row_0/A:col10/1731414291933/Put/seqid=0 2024-11-12T12:24:53,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414353119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414353121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414353129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414353136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741953_1129 (size=30955) 2024-11-12T12:24:53,167 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/009728bf53864216b9856d368c97326c 2024-11-12T12:24:53,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414353140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/299517d162044c3b9f33a774627cdc7f is 50, key is test_row_0/B:col10/1731414291933/Put/seqid=0 2024-11-12T12:24:53,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741954_1130 (size=12001) 2024-11-12T12:24:53,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414353252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414353252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414353252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414353271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414353292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,430 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T12:24:53,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414353462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414353462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414353462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414353482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414353511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,658 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/299517d162044c3b9f33a774627cdc7f 2024-11-12T12:24:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/55333adca56849eb9f2baad1e6db7b2b is 50, key is test_row_0/C:col10/1731414291933/Put/seqid=0 2024-11-12T12:24:53,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741955_1131 (size=12001) 2024-11-12T12:24:53,708 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/55333adca56849eb9f2baad1e6db7b2b 2024-11-12T12:24:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/009728bf53864216b9856d368c97326c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/009728bf53864216b9856d368c97326c 2024-11-12T12:24:53,720 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/009728bf53864216b9856d368c97326c, entries=150, sequenceid=41, filesize=30.2 K 2024-11-12T12:24:53,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/299517d162044c3b9f33a774627cdc7f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/299517d162044c3b9f33a774627cdc7f 2024-11-12T12:24:53,729 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/299517d162044c3b9f33a774627cdc7f, entries=150, sequenceid=41, filesize=11.7 K 2024-11-12T12:24:53,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/55333adca56849eb9f2baad1e6db7b2b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/55333adca56849eb9f2baad1e6db7b2b 2024-11-12T12:24:53,740 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/55333adca56849eb9f2baad1e6db7b2b, entries=150, sequenceid=41, filesize=11.7 K 2024-11-12T12:24:53,742 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for cfcbe6d6503fe59cbb8c8cb9085531ad in 721ms, sequenceid=41, compaction requested=false 2024-11-12T12:24:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-12T12:24:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-12T12:24:53,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-12T12:24:53,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9830 sec 2024-11-12T12:24:53,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9910 sec 2024-11-12T12:24:53,776 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:53,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-12T12:24:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:53,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111246033d5046514cb2a30cb58eba181351_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414293774/Put/seqid=0 2024-11-12T12:24:53,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414353828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414353830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414353832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414353839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414353842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741956_1132 (size=14594) 2024-11-12T12:24:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T12:24:53,902 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:53,903 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-12T12:24:53,906 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:53,909 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:53,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T12:24:53,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414353942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414353951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414353951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414353951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414353952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T12:24:54,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T12:24:54,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:54,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414354153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414354194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414354194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414354195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414354204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T12:24:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T12:24:54,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:54,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111246033d5046514cb2a30cb58eba181351_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111246033d5046514cb2a30cb58eba181351_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:54,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8e62132ba4da46efb95b6249576300b2, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:54,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8e62132ba4da46efb95b6249576300b2 is 175, key is test_row_0/A:col10/1731414293774/Put/seqid=0 2024-11-12T12:24:54,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741957_1133 (size=39549) 2024-11-12T12:24:54,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T12:24:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:54,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414354461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414354511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414354511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414354511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414354512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T12:24:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T12:24:54,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T12:24:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,691 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8e62132ba4da46efb95b6249576300b2 2024-11-12T12:24:54,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/540a98d0b0444033b93e186a90b781fa is 50, key is test_row_0/B:col10/1731414293774/Put/seqid=0 2024-11-12T12:24:54,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741958_1134 (size=12001) 2024-11-12T12:24:54,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T12:24:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414354971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:54,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:54,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T12:24:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:54,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:55,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414355021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414355021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414355021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:55,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414355032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T12:24:55,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T12:24:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:55,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:55,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:55,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/540a98d0b0444033b93e186a90b781fa 2024-11-12T12:24:55,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/e224a6cc622a4e928c0d9eeda5cc0932 is 50, key is test_row_0/C:col10/1731414293774/Put/seqid=0 2024-11-12T12:24:55,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741959_1135 (size=12001) 2024-11-12T12:24:55,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/e224a6cc622a4e928c0d9eeda5cc0932 2024-11-12T12:24:55,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8e62132ba4da46efb95b6249576300b2 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8e62132ba4da46efb95b6249576300b2 2024-11-12T12:24:55,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8e62132ba4da46efb95b6249576300b2, entries=200, sequenceid=54, filesize=38.6 K 2024-11-12T12:24:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/540a98d0b0444033b93e186a90b781fa as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/540a98d0b0444033b93e186a90b781fa 2024-11-12T12:24:55,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/540a98d0b0444033b93e186a90b781fa, entries=150, sequenceid=54, filesize=11.7 K 2024-11-12T12:24:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/e224a6cc622a4e928c0d9eeda5cc0932 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e224a6cc622a4e928c0d9eeda5cc0932 2024-11-12T12:24:55,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e224a6cc622a4e928c0d9eeda5cc0932, entries=150, sequenceid=54, filesize=11.7 K 2024-11-12T12:24:55,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1496ms, sequenceid=54, compaction requested=true 2024-11-12T12:24:55,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:55,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:55,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:55,273 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:55,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:55,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:55,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:55,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:55,273 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:55,274 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:55,275 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/B is initiating minor compaction (all files) 2024-11-12T12:24:55,275 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/B in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:55,275 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/33fec82b8bdc4a968b497313b1bc7697, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/299517d162044c3b9f33a774627cdc7f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/540a98d0b0444033b93e186a90b781fa] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=35.2 K 2024-11-12T12:24:55,275 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:55,275 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/A is initiating minor compaction (all files) 2024-11-12T12:24:55,275 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/A in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:55,276 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/b2b493879858450c901370a90f513902, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/009728bf53864216b9856d368c97326c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8e62132ba4da46efb95b6249576300b2] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=107.5 K 2024-11-12T12:24:55,276 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:55,276 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/b2b493879858450c901370a90f513902, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/009728bf53864216b9856d368c97326c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8e62132ba4da46efb95b6249576300b2] 2024-11-12T12:24:55,277 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33fec82b8bdc4a968b497313b1bc7697, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731414291763 2024-11-12T12:24:55,277 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting b2b493879858450c901370a90f513902, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731414291763 2024-11-12T12:24:55,277 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 299517d162044c3b9f33a774627cdc7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731414291922 2024-11-12T12:24:55,277 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 009728bf53864216b9856d368c97326c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731414291922 2024-11-12T12:24:55,278 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 540a98d0b0444033b93e186a90b781fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731414293117 2024-11-12T12:24:55,278 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e62132ba4da46efb95b6249576300b2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731414293117 2024-11-12T12:24:55,292 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:55,292 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/88799061b50444a7a38b8a744380df83 is 50, key is test_row_0/B:col10/1731414293774/Put/seqid=0 2024-11-12T12:24:55,301 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:55,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T12:24:55,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:55,305 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-12T12:24:55,325 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112ed880302d116455ab883b229b3aa452f_cfcbe6d6503fe59cbb8c8cb9085531ad store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:55,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741960_1136 (size=12104) 2024-11-12T12:24:55,333 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112ed880302d116455ab883b229b3aa452f_cfcbe6d6503fe59cbb8c8cb9085531ad, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:55,333 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ed880302d116455ab883b229b3aa452f_cfcbe6d6503fe59cbb8c8cb9085531ad because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:55,355 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/88799061b50444a7a38b8a744380df83 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/88799061b50444a7a38b8a744380df83 2024-11-12T12:24:55,362 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/B of cfcbe6d6503fe59cbb8c8cb9085531ad into 88799061b50444a7a38b8a744380df83(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:55,362 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:55,362 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/B, priority=13, startTime=1731414295273; duration=0sec 2024-11-12T12:24:55,362 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:55,362 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:B 2024-11-12T12:24:55,362 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:55,363 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:55,363 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/C is initiating minor compaction (all files) 2024-11-12T12:24:55,364 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/C in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:55,364 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/50e2a1ef938841978f4347aa7ead70d6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/55333adca56849eb9f2baad1e6db7b2b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e224a6cc622a4e928c0d9eeda5cc0932] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=35.2 K 2024-11-12T12:24:55,364 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50e2a1ef938841978f4347aa7ead70d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731414291763 2024-11-12T12:24:55,365 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55333adca56849eb9f2baad1e6db7b2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731414291922 2024-11-12T12:24:55,365 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting e224a6cc622a4e928c0d9eeda5cc0932, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731414293117 2024-11-12T12:24:55,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111242498afcc10f44a4b6e8e449923d5f1f_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414293827/Put/seqid=0 2024-11-12T12:24:55,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741961_1137 (size=4469) 2024-11-12T12:24:55,389 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#C#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:55,390 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/619f8b3732794312bdc3a956f58018f2 is 50, key is test_row_0/C:col10/1731414293774/Put/seqid=0 2024-11-12T12:24:55,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741962_1138 (size=12154) 2024-11-12T12:24:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:55,405 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111242498afcc10f44a4b6e8e449923d5f1f_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111242498afcc10f44a4b6e8e449923d5f1f_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:55,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8cdd2016375b4f5892c25a6ad12ec124, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8cdd2016375b4f5892c25a6ad12ec124 is 175, key is test_row_0/A:col10/1731414293827/Put/seqid=0 2024-11-12T12:24:55,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741963_1139 (size=12104) 2024-11-12T12:24:55,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741964_1140 (size=30955) 2024-11-12T12:24:55,419 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/619f8b3732794312bdc3a956f58018f2 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/619f8b3732794312bdc3a956f58018f2 2024-11-12T12:24:55,433 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/C of cfcbe6d6503fe59cbb8c8cb9085531ad into 619f8b3732794312bdc3a956f58018f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:55,433 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:55,433 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/C, priority=13, startTime=1731414295273; duration=0sec 2024-11-12T12:24:55,434 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:55,434 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:C 2024-11-12T12:24:55,789 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#A#compaction#118 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:55,789 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/33daf090080c4026bddc7f410f9bb255 is 175, key is test_row_0/A:col10/1731414293774/Put/seqid=0 2024-11-12T12:24:55,816 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8cdd2016375b4f5892c25a6ad12ec124 2024-11-12T12:24:55,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741965_1141 (size=31058) 2024-11-12T12:24:55,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/71bb0aa4220049a1aa721514fb09b8d5 is 50, key is test_row_0/B:col10/1731414293827/Put/seqid=0 2024-11-12T12:24:55,862 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/33daf090080c4026bddc7f410f9bb255 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/33daf090080c4026bddc7f410f9bb255 2024-11-12T12:24:55,870 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/A of cfcbe6d6503fe59cbb8c8cb9085531ad into 33daf090080c4026bddc7f410f9bb255(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:55,871 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:55,871 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/A, priority=13, startTime=1731414295273; duration=0sec 2024-11-12T12:24:55,871 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:55,871 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:A 2024-11-12T12:24:55,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741966_1142 (size=12001) 2024-11-12T12:24:55,892 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/71bb0aa4220049a1aa721514fb09b8d5 2024-11-12T12:24:55,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/6743a08f9e944d15b62a945d1f3294cd is 50, key is test_row_0/C:col10/1731414293827/Put/seqid=0 2024-11-12T12:24:55,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741967_1143 (size=12001) 2024-11-12T12:24:55,965 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/6743a08f9e944d15b62a945d1f3294cd 2024-11-12T12:24:55,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8cdd2016375b4f5892c25a6ad12ec124 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8cdd2016375b4f5892c25a6ad12ec124 2024-11-12T12:24:55,983 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8cdd2016375b4f5892c25a6ad12ec124, entries=150, sequenceid=78, filesize=30.2 K 2024-11-12T12:24:55,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/71bb0aa4220049a1aa721514fb09b8d5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/71bb0aa4220049a1aa721514fb09b8d5 2024-11-12T12:24:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:55,998 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/71bb0aa4220049a1aa721514fb09b8d5, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T12:24:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/6743a08f9e944d15b62a945d1f3294cd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/6743a08f9e944d15b62a945d1f3294cd 2024-11-12T12:24:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,011 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/6743a08f9e944d15b62a945d1f3294cd, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T12:24:56,012 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for cfcbe6d6503fe59cbb8c8cb9085531ad in 707ms, sequenceid=78, compaction requested=false 2024-11-12T12:24:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-12T12:24:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-12T12:24:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-12T12:24:56,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1070 sec 2024-11-12T12:24:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.1200 sec 2024-11-12T12:24:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T12:24:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,061 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-12T12:24:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T12:24:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,065 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,067 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:56,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T12:24:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,196 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:56,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:24:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T12:24:56,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:56,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127f7bdf7b60ca4d709f2bf2dfb91585a1_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414296191/Put/seqid=0 2024-11-12T12:24:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741969_1145 (size=24358) 2024-11-12T12:24:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,328 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127f7bdf7b60ca4d709f2bf2dfb91585a1_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127f7bdf7b60ca4d709f2bf2dfb91585a1_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,330 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2489ada82d6e48fd867072c658106569, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2489ada82d6e48fd867072c658106569 is 175, key is test_row_0/A:col10/1731414296191/Put/seqid=0 2024-11-12T12:24:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741968_1144 (size=73995) 2024-11-12T12:24:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:56,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T12:24:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T12:24:56,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414356376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414356345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414356380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414356381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414356382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414356501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414356502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414356502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414356502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414356513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T12:24:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T12:24:56,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T12:24:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414356721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414356722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414356722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414356722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414356722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:56,735 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2489ada82d6e48fd867072c658106569 2024-11-12T12:24:56,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/2d1536b2bf5a4d4eb654ecf7320338a3 is 50, key is test_row_0/B:col10/1731414296191/Put/seqid=0 2024-11-12T12:24:56,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741970_1146 (size=12001) 2024-11-12T12:24:56,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/2d1536b2bf5a4d4eb654ecf7320338a3 2024-11-12T12:24:56,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/4b073f3124794a5f8c2ddad746bc08ec is 50, key is test_row_0/C:col10/1731414296191/Put/seqid=0 2024-11-12T12:24:56,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T12:24:56,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:56,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:56,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:56,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741971_1147 (size=12001) 2024-11-12T12:24:57,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T12:24:57,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:57,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:57,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:57,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414357031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414357031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414357041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414357041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414357041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T12:24:57,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:57,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:57,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T12:24:57,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/4b073f3124794a5f8c2ddad746bc08ec 2024-11-12T12:24:57,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2489ada82d6e48fd867072c658106569 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2489ada82d6e48fd867072c658106569 2024-11-12T12:24:57,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T12:24:57,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:57,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:57,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:57,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2489ada82d6e48fd867072c658106569, entries=400, sequenceid=92, filesize=72.3 K 2024-11-12T12:24:57,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/2d1536b2bf5a4d4eb654ecf7320338a3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/2d1536b2bf5a4d4eb654ecf7320338a3 2024-11-12T12:24:57,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/2d1536b2bf5a4d4eb654ecf7320338a3, entries=150, sequenceid=92, filesize=11.7 K 2024-11-12T12:24:57,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/4b073f3124794a5f8c2ddad746bc08ec as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4b073f3124794a5f8c2ddad746bc08ec 2024-11-12T12:24:57,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4b073f3124794a5f8c2ddad746bc08ec, entries=150, sequenceid=92, filesize=11.7 K 2024-11-12T12:24:57,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1157ms, sequenceid=92, compaction requested=true 2024-11-12T12:24:57,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:57,354 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:57,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:57,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:57,355 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:57,355 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136008 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:57,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:57,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:57,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:57,355 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/A is initiating minor compaction (all files) 2024-11-12T12:24:57,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:57,355 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/A in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,356 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/33daf090080c4026bddc7f410f9bb255, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8cdd2016375b4f5892c25a6ad12ec124, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2489ada82d6e48fd867072c658106569] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=132.8 K 2024-11-12T12:24:57,356 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,356 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/33daf090080c4026bddc7f410f9bb255, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8cdd2016375b4f5892c25a6ad12ec124, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2489ada82d6e48fd867072c658106569] 2024-11-12T12:24:57,357 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33daf090080c4026bddc7f410f9bb255, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731414293117 2024-11-12T12:24:57,359 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:57,359 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/B is initiating minor compaction (all files) 2024-11-12T12:24:57,359 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/B in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,359 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/88799061b50444a7a38b8a744380df83, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/71bb0aa4220049a1aa721514fb09b8d5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/2d1536b2bf5a4d4eb654ecf7320338a3] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=35.3 K 2024-11-12T12:24:57,359 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cdd2016375b4f5892c25a6ad12ec124, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731414293826 2024-11-12T12:24:57,361 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2489ada82d6e48fd867072c658106569, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731414296145 2024-11-12T12:24:57,361 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 88799061b50444a7a38b8a744380df83, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731414293117 2024-11-12T12:24:57,363 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 71bb0aa4220049a1aa721514fb09b8d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731414293826 2024-11-12T12:24:57,364 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d1536b2bf5a4d4eb654ecf7320338a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731414296160 2024-11-12T12:24:57,386 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:57,399 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#B#compaction#127 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:57,399 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a7cf3edb9b174766b2c257a0616907ac is 50, key is test_row_0/B:col10/1731414296191/Put/seqid=0 2024-11-12T12:24:57,410 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112164e4c580e134a98afb7e065b9c9fb85_cfcbe6d6503fe59cbb8c8cb9085531ad store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:57,414 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112164e4c580e134a98afb7e065b9c9fb85_cfcbe6d6503fe59cbb8c8cb9085531ad, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:57,415 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112164e4c580e134a98afb7e065b9c9fb85_cfcbe6d6503fe59cbb8c8cb9085531ad because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741973_1149 (size=4469) 2024-11-12T12:24:57,444 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#A#compaction#126 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:57,444 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/3e741074918448d58d0852faab1841be is 175, key is test_row_0/A:col10/1731414296191/Put/seqid=0 2024-11-12T12:24:57,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T12:24:57,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741972_1148 (size=12207) 2024-11-12T12:24:57,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,469 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:24:57,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111230198413dd6c470f8606cda4b39b039e_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414296378/Put/seqid=0 2024-11-12T12:24:57,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741974_1150 (size=31161) 2024-11-12T12:24:57,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:57,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741975_1151 (size=12154) 2024-11-12T12:24:57,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:57,568 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111230198413dd6c470f8606cda4b39b039e_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111230198413dd6c470f8606cda4b39b039e_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/c44ca2b63bb344b1aa6fc2c07a911614, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/c44ca2b63bb344b1aa6fc2c07a911614 is 175, key is test_row_0/A:col10/1731414296378/Put/seqid=0 2024-11-12T12:24:57,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414357577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414357581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414357581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414357586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414357587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741976_1152 (size=30955) 2024-11-12T12:24:57,622 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/c44ca2b63bb344b1aa6fc2c07a911614 2024-11-12T12:24:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/97f2d91ebb084540baa7dfc520a8a047 is 50, key is test_row_0/B:col10/1731414296378/Put/seqid=0 2024-11-12T12:24:57,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741977_1153 (size=12001) 2024-11-12T12:24:57,692 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/97f2d91ebb084540baa7dfc520a8a047 2024-11-12T12:24:57,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414357691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/fc02f33b8949402ab5bbc0bc83725a43 is 50, key is test_row_0/C:col10/1731414296378/Put/seqid=0 2024-11-12T12:24:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414357701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414357701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414357701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414357701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741978_1154 (size=12001) 2024-11-12T12:24:57,879 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a7cf3edb9b174766b2c257a0616907ac as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a7cf3edb9b174766b2c257a0616907ac 2024-11-12T12:24:57,885 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/B of cfcbe6d6503fe59cbb8c8cb9085531ad into a7cf3edb9b174766b2c257a0616907ac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:57,885 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:57,886 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/B, priority=13, startTime=1731414297355; duration=0sec 2024-11-12T12:24:57,886 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:57,886 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:B 2024-11-12T12:24:57,886 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:57,887 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:57,887 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/C is initiating minor compaction (all files) 2024-11-12T12:24:57,888 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/C in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:57,888 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/619f8b3732794312bdc3a956f58018f2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/6743a08f9e944d15b62a945d1f3294cd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4b073f3124794a5f8c2ddad746bc08ec] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=35.3 K 2024-11-12T12:24:57,888 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 619f8b3732794312bdc3a956f58018f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731414293117 2024-11-12T12:24:57,889 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6743a08f9e944d15b62a945d1f3294cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731414293826 2024-11-12T12:24:57,889 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b073f3124794a5f8c2ddad746bc08ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731414296160 2024-11-12T12:24:57,905 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:57,906 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/09c0ebee7aeb43f2bacc3b0f7ba5c5a4 is 50, key is test_row_0/C:col10/1731414296191/Put/seqid=0 2024-11-12T12:24:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414357911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,925 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/3e741074918448d58d0852faab1841be as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/3e741074918448d58d0852faab1841be 2024-11-12T12:24:57,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414357922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414357922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414357922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414357922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:57,933 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/A of cfcbe6d6503fe59cbb8c8cb9085531ad into 3e741074918448d58d0852faab1841be(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:57,933 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:57,933 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/A, priority=13, startTime=1731414297354; duration=0sec 2024-11-12T12:24:57,933 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:57,933 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:A 2024-11-12T12:24:57,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741979_1155 (size=12207) 2024-11-12T12:24:57,962 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/09c0ebee7aeb43f2bacc3b0f7ba5c5a4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/09c0ebee7aeb43f2bacc3b0f7ba5c5a4 2024-11-12T12:24:57,970 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/C of cfcbe6d6503fe59cbb8c8cb9085531ad into 09c0ebee7aeb43f2bacc3b0f7ba5c5a4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:57,970 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:57,970 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/C, priority=13, startTime=1731414297355; duration=0sec 2024-11-12T12:24:57,970 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:57,970 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:C 2024-11-12T12:24:58,144 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/fc02f33b8949402ab5bbc0bc83725a43 2024-11-12T12:24:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/c44ca2b63bb344b1aa6fc2c07a911614 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/c44ca2b63bb344b1aa6fc2c07a911614 2024-11-12T12:24:58,161 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/c44ca2b63bb344b1aa6fc2c07a911614, entries=150, sequenceid=117, filesize=30.2 K 2024-11-12T12:24:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/97f2d91ebb084540baa7dfc520a8a047 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/97f2d91ebb084540baa7dfc520a8a047 2024-11-12T12:24:58,173 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/97f2d91ebb084540baa7dfc520a8a047, entries=150, sequenceid=117, filesize=11.7 K 2024-11-12T12:24:58,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/fc02f33b8949402ab5bbc0bc83725a43 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/fc02f33b8949402ab5bbc0bc83725a43 2024-11-12T12:24:58,181 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/fc02f33b8949402ab5bbc0bc83725a43, entries=150, sequenceid=117, filesize=11.7 K 2024-11-12T12:24:58,182 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for cfcbe6d6503fe59cbb8c8cb9085531ad in 713ms, sequenceid=117, compaction requested=false 2024-11-12T12:24:58,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:58,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-12T12:24:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-12T12:24:58,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-12T12:24:58,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1170 sec 2024-11-12T12:24:58,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1260 sec 2024-11-12T12:24:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T12:24:58,212 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:24:58,213 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:24:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-12T12:24:58,216 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:24:58,218 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:24:58,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:24:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T12:24:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:58,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-12T12:24:58,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e4be8f7312234714af95e4e31a9203fe_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414298225/Put/seqid=0 2024-11-12T12:24:58,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414358265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414358267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414358269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414358274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414358276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741980_1156 (size=14644) 2024-11-12T12:24:58,291 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:58,301 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e4be8f7312234714af95e4e31a9203fe_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e4be8f7312234714af95e4e31a9203fe_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:58,306 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2a2afc43e52743fdb273855e22dfa2bc, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:58,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2a2afc43e52743fdb273855e22dfa2bc is 175, key is test_row_0/A:col10/1731414298225/Put/seqid=0 2024-11-12T12:24:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T12:24:58,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741981_1157 (size=39599) 2024-11-12T12:24:58,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:58,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414358382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414358382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414358382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414358392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414358392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:58,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:58,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T12:24:58,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414358602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414358602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414358602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414358602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414358602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:58,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:58,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,759 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2a2afc43e52743fdb273855e22dfa2bc 2024-11-12T12:24:58,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a76e0b68e571445a9b96bfea44926179 is 50, key is test_row_0/B:col10/1731414298225/Put/seqid=0 2024-11-12T12:24:58,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741982_1158 (size=12051) 2024-11-12T12:24:58,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a76e0b68e571445a9b96bfea44926179 2024-11-12T12:24:58,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/f04c92eb020a4db2bdf55fd9565878f2 is 50, key is test_row_0/C:col10/1731414298225/Put/seqid=0 2024-11-12T12:24:58,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:58,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:58,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T12:24:58,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741983_1159 (size=12051) 2024-11-12T12:24:58,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414358911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414358911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414358911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414358921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414358922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:58,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:58,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:59,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/f04c92eb020a4db2bdf55fd9565878f2 2024-11-12T12:24:59,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2a2afc43e52743fdb273855e22dfa2bc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2a2afc43e52743fdb273855e22dfa2bc 2024-11-12T12:24:59,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2a2afc43e52743fdb273855e22dfa2bc, entries=200, sequenceid=133, filesize=38.7 K 2024-11-12T12:24:59,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a76e0b68e571445a9b96bfea44926179 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a76e0b68e571445a9b96bfea44926179 2024-11-12T12:24:59,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a76e0b68e571445a9b96bfea44926179, entries=150, sequenceid=133, filesize=11.8 K 2024-11-12T12:24:59,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/f04c92eb020a4db2bdf55fd9565878f2 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f04c92eb020a4db2bdf55fd9565878f2 2024-11-12T12:24:59,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:59,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:59,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f04c92eb020a4db2bdf55fd9565878f2, entries=150, sequenceid=133, filesize=11.8 K 2024-11-12T12:24:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1078ms, sequenceid=133, compaction requested=true 2024-11-12T12:24:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:24:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:59,306 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:24:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:24:59,306 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:24:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:59,307 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101715 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:59,307 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/A is initiating minor compaction (all files) 2024-11-12T12:24:59,307 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:59,307 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/A in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,307 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/B is initiating minor compaction (all files) 2024-11-12T12:24:59,307 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/B in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,308 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/3e741074918448d58d0852faab1841be, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/c44ca2b63bb344b1aa6fc2c07a911614, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2a2afc43e52743fdb273855e22dfa2bc] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=99.3 K 2024-11-12T12:24:59,308 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a7cf3edb9b174766b2c257a0616907ac, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/97f2d91ebb084540baa7dfc520a8a047, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a76e0b68e571445a9b96bfea44926179] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=35.4 K 2024-11-12T12:24:59,308 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,308 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/3e741074918448d58d0852faab1841be, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/c44ca2b63bb344b1aa6fc2c07a911614, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2a2afc43e52743fdb273855e22dfa2bc] 2024-11-12T12:24:59,308 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a7cf3edb9b174766b2c257a0616907ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731414296160 2024-11-12T12:24:59,308 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e741074918448d58d0852faab1841be, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731414296160 2024-11-12T12:24:59,309 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f2d91ebb084540baa7dfc520a8a047, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731414296344 2024-11-12T12:24:59,309 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting c44ca2b63bb344b1aa6fc2c07a911614, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731414296344 2024-11-12T12:24:59,309 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a76e0b68e571445a9b96bfea44926179, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414297580 2024-11-12T12:24:59,309 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a2afc43e52743fdb273855e22dfa2bc, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414297578 2024-11-12T12:24:59,338 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:59,339 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/3fd8dd31201a47db92f5eda265f2aafd is 50, key is test_row_0/B:col10/1731414298225/Put/seqid=0 2024-11-12T12:24:59,351 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T12:24:59,385 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411123e6077a0121c4d949d3eb5af596ca6bd_cfcbe6d6503fe59cbb8c8cb9085531ad store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:59,387 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411123e6077a0121c4d949d3eb5af596ca6bd_cfcbe6d6503fe59cbb8c8cb9085531ad, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:59,388 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123e6077a0121c4d949d3eb5af596ca6bd_cfcbe6d6503fe59cbb8c8cb9085531ad because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:59,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741984_1160 (size=12359) 2024-11-12T12:24:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:59,427 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:24:59,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-12T12:24:59,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741985_1161 (size=4469) 2024-11-12T12:24:59,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d8ddce526b2a461dbd2c9ce178bb3b52_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414298265/Put/seqid=0 2024-11-12T12:24:59,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414359447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414359454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414359454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741986_1162 (size=14794) 2024-11-12T12:24:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414359456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414359457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414359561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414359561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414359562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414359571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414359571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:59,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:59,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414359772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414359772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414359772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414359781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:24:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414359781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:24:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:24:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:24:59,821 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/3fd8dd31201a47db92f5eda265f2aafd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/3fd8dd31201a47db92f5eda265f2aafd 2024-11-12T12:24:59,829 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/B of cfcbe6d6503fe59cbb8c8cb9085531ad into 3fd8dd31201a47db92f5eda265f2aafd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:24:59,829 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:24:59,829 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/B, priority=13, startTime=1731414299306; duration=0sec 2024-11-12T12:24:59,829 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:24:59,829 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:B 2024-11-12T12:24:59,830 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:24:59,831 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:24:59,831 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/C is initiating minor compaction (all files) 2024-11-12T12:24:59,831 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/C in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,831 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/09c0ebee7aeb43f2bacc3b0f7ba5c5a4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/fc02f33b8949402ab5bbc0bc83725a43, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f04c92eb020a4db2bdf55fd9565878f2] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=35.4 K 2024-11-12T12:24:59,832 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c0ebee7aeb43f2bacc3b0f7ba5c5a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731414296160 2024-11-12T12:24:59,832 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting fc02f33b8949402ab5bbc0bc83725a43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731414296344 2024-11-12T12:24:59,833 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting f04c92eb020a4db2bdf55fd9565878f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414297580 2024-11-12T12:24:59,853 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#A#compaction#136 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:59,854 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/e55006ad240f43428ded0ef55a7b0466 is 175, key is test_row_0/A:col10/1731414298225/Put/seqid=0 2024-11-12T12:24:59,867 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:24:59,868 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/28ea0fb4d9dd48c8ae5e6f30da8c4d2e is 50, key is test_row_0/C:col10/1731414298225/Put/seqid=0 2024-11-12T12:24:59,872 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:24:59,877 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d8ddce526b2a461dbd2c9ce178bb3b52_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d8ddce526b2a461dbd2c9ce178bb3b52_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:24:59,878 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/52274911d3ef4d8ea27fc226bc2337af, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:24:59,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/52274911d3ef4d8ea27fc226bc2337af is 175, key is test_row_0/A:col10/1731414298265/Put/seqid=0 2024-11-12T12:24:59,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741987_1163 (size=31313) 2024-11-12T12:24:59,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:24:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:24:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:24:59,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:24:59,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741988_1164 (size=39749) 2024-11-12T12:24:59,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741989_1165 (size=12359) 2024-11-12T12:25:00,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:00,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:00,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414360092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414360092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414360092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414360092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414360092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:00,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,309 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/e55006ad240f43428ded0ef55a7b0466 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/e55006ad240f43428ded0ef55a7b0466 2024-11-12T12:25:00,322 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/A of cfcbe6d6503fe59cbb8c8cb9085531ad into e55006ad240f43428ded0ef55a7b0466(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-12T12:25:00,322 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:00,322 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/A, priority=13, startTime=1731414299306; duration=1sec 2024-11-12T12:25:00,322 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:00,322 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:A 2024-11-12T12:25:00,338 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/52274911d3ef4d8ea27fc226bc2337af 2024-11-12T12:25:00,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/1e02330556ca42b7960361c4ae3e230e is 50, key is test_row_0/B:col10/1731414298265/Put/seqid=0 2024-11-12T12:25:00,380 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/28ea0fb4d9dd48c8ae5e6f30da8c4d2e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/28ea0fb4d9dd48c8ae5e6f30da8c4d2e 2024-11-12T12:25:00,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T12:25:00,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,389 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/C of cfcbe6d6503fe59cbb8c8cb9085531ad into 28ea0fb4d9dd48c8ae5e6f30da8c4d2e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:00,389 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:00,389 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/C, priority=13, startTime=1731414299306; duration=0sec 2024-11-12T12:25:00,389 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:00,389 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:C 2024-11-12T12:25:00,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741990_1166 (size=12151) 2024-11-12T12:25:00,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414360601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414360602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414360611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414360611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414360611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:00,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/1e02330556ca42b7960361c4ae3e230e 2024-11-12T12:25:00,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/ac442f3b28594b49b44455b2a1c80a14 is 50, key is test_row_0/C:col10/1731414298265/Put/seqid=0 2024-11-12T12:25:00,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:00,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:00,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741991_1167 (size=12151) 2024-11-12T12:25:00,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:00,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:00,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:01,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:01,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:01,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:01,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:01,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:01,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:01,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/ac442f3b28594b49b44455b2a1c80a14 2024-11-12T12:25:01,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/52274911d3ef4d8ea27fc226bc2337af as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/52274911d3ef4d8ea27fc226bc2337af 2024-11-12T12:25:01,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/52274911d3ef4d8ea27fc226bc2337af, entries=200, sequenceid=157, filesize=38.8 K 2024-11-12T12:25:01,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/1e02330556ca42b7960361c4ae3e230e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1e02330556ca42b7960361c4ae3e230e 2024-11-12T12:25:01,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1e02330556ca42b7960361c4ae3e230e, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T12:25:01,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/ac442f3b28594b49b44455b2a1c80a14 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/ac442f3b28594b49b44455b2a1c80a14 2024-11-12T12:25:01,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:01,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:01,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:01,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:01,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:01,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:01,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/ac442f3b28594b49b44455b2a1c80a14, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T12:25:01,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1880ms, sequenceid=157, compaction requested=false 2024-11-12T12:25:01,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:01,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T12:25:01,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:01,458 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-12T12:25:01,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127e13b55dd22f462bab705ce16f63467c_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414299438/Put/seqid=0 2024-11-12T12:25:01,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741992_1168 (size=12304) 2024-11-12T12:25:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:01,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:01,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414361645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414361646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414361648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414361649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414361653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414361761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414361761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414361761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414361761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414361761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:01,920 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127e13b55dd22f462bab705ce16f63467c_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127e13b55dd22f462bab705ce16f63467c_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:01,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8666501b8933482ca03828e5ebe5a524, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8666501b8933482ca03828e5ebe5a524 is 175, key is test_row_0/A:col10/1731414299438/Put/seqid=0 2024-11-12T12:25:01,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741993_1169 (size=31105) 2024-11-12T12:25:01,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414361971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414361971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414361972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414361972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414361972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,071 INFO [master/4a91c05c96a4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T12:25:02,071 INFO [master/4a91c05c96a4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T12:25:02,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414362291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414362291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414362292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414362292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414362292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,331 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8666501b8933482ca03828e5ebe5a524 2024-11-12T12:25:02,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/ddbb57f1e8c44d6497b6b86f1145af7f is 50, key is test_row_0/B:col10/1731414299438/Put/seqid=0 2024-11-12T12:25:02,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741994_1170 (size=12151) 2024-11-12T12:25:02,375 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/ddbb57f1e8c44d6497b6b86f1145af7f 2024-11-12T12:25:02,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/f65b0292928048aeabbdfb661ccf32fd is 50, key is test_row_0/C:col10/1731414299438/Put/seqid=0 2024-11-12T12:25:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T12:25:02,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741995_1171 (size=12151) 2024-11-12T12:25:02,411 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/f65b0292928048aeabbdfb661ccf32fd 2024-11-12T12:25:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/8666501b8933482ca03828e5ebe5a524 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8666501b8933482ca03828e5ebe5a524 2024-11-12T12:25:02,439 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8666501b8933482ca03828e5ebe5a524, entries=150, sequenceid=173, filesize=30.4 K 2024-11-12T12:25:02,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/ddbb57f1e8c44d6497b6b86f1145af7f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ddbb57f1e8c44d6497b6b86f1145af7f 2024-11-12T12:25:02,447 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ddbb57f1e8c44d6497b6b86f1145af7f, entries=150, sequenceid=173, filesize=11.9 K 2024-11-12T12:25:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/f65b0292928048aeabbdfb661ccf32fd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f65b0292928048aeabbdfb661ccf32fd 2024-11-12T12:25:02,458 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f65b0292928048aeabbdfb661ccf32fd, entries=150, sequenceid=173, filesize=11.9 K 2024-11-12T12:25:02,460 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1002ms, sequenceid=173, compaction requested=true 2024-11-12T12:25:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-12T12:25:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-12T12:25:02,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-12T12:25:02,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2450 sec 2024-11-12T12:25:02,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 4.2540 sec 2024-11-12T12:25:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:02,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-12T12:25:02,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dec9c90414464a98ab2b5587c7c886a7_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414301649/Put/seqid=0 2024-11-12T12:25:02,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414362825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414362826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414362828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414362833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414362833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741996_1172 (size=12304) 2024-11-12T12:25:02,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:02,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dec9c90414464a98ab2b5587c7c886a7_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112dec9c90414464a98ab2b5587c7c886a7_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:02,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/421694d1398843b69ebd569b2af91e23, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:02,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/421694d1398843b69ebd569b2af91e23 is 175, key is test_row_0/A:col10/1731414301649/Put/seqid=0 2024-11-12T12:25:02,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741997_1173 (size=31105) 2024-11-12T12:25:02,897 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/421694d1398843b69ebd569b2af91e23 2024-11-12T12:25:02,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a2d4f05870c8492684d357ab9ce76e9c is 50, key is test_row_0/B:col10/1731414301649/Put/seqid=0 2024-11-12T12:25:02,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741998_1174 (size=12151) 2024-11-12T12:25:02,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a2d4f05870c8492684d357ab9ce76e9c 2024-11-12T12:25:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414362941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414362941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414362941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/57e69806a5bd4b2fa321305141aba0d8 is 50, key is test_row_0/C:col10/1731414301649/Put/seqid=0 2024-11-12T12:25:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414362941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414362979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:02,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741999_1175 (size=12151) 2024-11-12T12:25:03,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414363181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414363191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414363192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414363192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414363192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/57e69806a5bd4b2fa321305141aba0d8 2024-11-12T12:25:03,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/421694d1398843b69ebd569b2af91e23 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/421694d1398843b69ebd569b2af91e23 2024-11-12T12:25:03,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/421694d1398843b69ebd569b2af91e23, entries=150, sequenceid=198, filesize=30.4 K 2024-11-12T12:25:03,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a2d4f05870c8492684d357ab9ce76e9c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a2d4f05870c8492684d357ab9ce76e9c 2024-11-12T12:25:03,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a2d4f05870c8492684d357ab9ce76e9c, entries=150, sequenceid=198, filesize=11.9 K 2024-11-12T12:25:03,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/57e69806a5bd4b2fa321305141aba0d8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/57e69806a5bd4b2fa321305141aba0d8 2024-11-12T12:25:03,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/57e69806a5bd4b2fa321305141aba0d8, entries=150, sequenceid=198, filesize=11.9 K 2024-11-12T12:25:03,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for cfcbe6d6503fe59cbb8c8cb9085531ad in 609ms, sequenceid=198, compaction requested=true 2024-11-12T12:25:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:03,423 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T12:25:03,423 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:03,430 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133272 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:03,430 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/A is initiating minor compaction (all files) 2024-11-12T12:25:03,430 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/A in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:03,430 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/e55006ad240f43428ded0ef55a7b0466, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/52274911d3ef4d8ea27fc226bc2337af, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8666501b8933482ca03828e5ebe5a524, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/421694d1398843b69ebd569b2af91e23] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=130.1 K 2024-11-12T12:25:03,430 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:03,431 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/e55006ad240f43428ded0ef55a7b0466, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/52274911d3ef4d8ea27fc226bc2337af, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8666501b8933482ca03828e5ebe5a524, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/421694d1398843b69ebd569b2af91e23] 2024-11-12T12:25:03,431 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:03,431 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/B is initiating minor compaction (all files) 2024-11-12T12:25:03,431 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/B in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:03,431 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/3fd8dd31201a47db92f5eda265f2aafd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1e02330556ca42b7960361c4ae3e230e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ddbb57f1e8c44d6497b6b86f1145af7f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a2d4f05870c8492684d357ab9ce76e9c] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=47.7 K 2024-11-12T12:25:03,432 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd8dd31201a47db92f5eda265f2aafd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414297580 2024-11-12T12:25:03,432 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting e55006ad240f43428ded0ef55a7b0466, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414297580 2024-11-12T12:25:03,433 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e02330556ca42b7960361c4ae3e230e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731414298265 2024-11-12T12:25:03,433 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ddbb57f1e8c44d6497b6b86f1145af7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731414299438 2024-11-12T12:25:03,434 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d4f05870c8492684d357ab9ce76e9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731414301646 2024-11-12T12:25:03,434 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52274911d3ef4d8ea27fc226bc2337af, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731414298265 2024-11-12T12:25:03,434 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8666501b8933482ca03828e5ebe5a524, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731414299438 2024-11-12T12:25:03,435 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 421694d1398843b69ebd569b2af91e23, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731414301646 2024-11-12T12:25:03,447 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:03,453 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:03,454 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/cb6851bc22e54e88a91d89ba5fb69b7a is 50, key is test_row_0/B:col10/1731414301649/Put/seqid=0 2024-11-12T12:25:03,460 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112f7a5b04df810442f80d8d99ee805cd7b_cfcbe6d6503fe59cbb8c8cb9085531ad store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:03,462 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112f7a5b04df810442f80d8d99ee805cd7b_cfcbe6d6503fe59cbb8c8cb9085531ad, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:03,463 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f7a5b04df810442f80d8d99ee805cd7b_cfcbe6d6503fe59cbb8c8cb9085531ad because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:03,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742000_1176 (size=12595) 2024-11-12T12:25:03,484 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/cb6851bc22e54e88a91d89ba5fb69b7a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/cb6851bc22e54e88a91d89ba5fb69b7a 2024-11-12T12:25:03,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742001_1177 (size=4469) 2024-11-12T12:25:03,487 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#A#compaction#147 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:03,488 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/7f0be1ad39c6455292ad8e471baa5ba6 is 175, key is test_row_0/A:col10/1731414301649/Put/seqid=0 2024-11-12T12:25:03,492 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/B of cfcbe6d6503fe59cbb8c8cb9085531ad into cb6851bc22e54e88a91d89ba5fb69b7a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:03,492 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:03,492 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/B, priority=12, startTime=1731414303423; duration=0sec 2024-11-12T12:25:03,492 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:03,492 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:B 2024-11-12T12:25:03,492 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:03,495 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:03,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-12T12:25:03,497 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:03,497 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/C is initiating minor compaction (all files) 2024-11-12T12:25:03,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742002_1178 (size=31549) 2024-11-12T12:25:03,497 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/C in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:03,497 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/28ea0fb4d9dd48c8ae5e6f30da8c4d2e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/ac442f3b28594b49b44455b2a1c80a14, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f65b0292928048aeabbdfb661ccf32fd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/57e69806a5bd4b2fa321305141aba0d8] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=47.7 K 2024-11-12T12:25:03,499 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ea0fb4d9dd48c8ae5e6f30da8c4d2e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414297580 2024-11-12T12:25:03,500 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ac442f3b28594b49b44455b2a1c80a14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731414298265 2024-11-12T12:25:03,502 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting f65b0292928048aeabbdfb661ccf32fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731414299438 2024-11-12T12:25:03,503 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e69806a5bd4b2fa321305141aba0d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731414301646 2024-11-12T12:25:03,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111228e861f288d641f0a5d782fe5f232be4_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414303494/Put/seqid=0 2024-11-12T12:25:03,526 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#C#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:03,527 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/e80590448c64411da4d0ee52e1eb1441 is 50, key is test_row_0/C:col10/1731414301649/Put/seqid=0 2024-11-12T12:25:03,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414363531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742004_1180 (size=12595) 2024-11-12T12:25:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414363536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414363538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414363539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414363540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742003_1179 (size=14794) 2024-11-12T12:25:03,548 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:03,552 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/e80590448c64411da4d0ee52e1eb1441 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e80590448c64411da4d0ee52e1eb1441 2024-11-12T12:25:03,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111228e861f288d641f0a5d782fe5f232be4_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111228e861f288d641f0a5d782fe5f232be4_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:03,557 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/66e5878b13994e49bcd5c7b342cdf079, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:03,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/66e5878b13994e49bcd5c7b342cdf079 is 175, key is test_row_0/A:col10/1731414303494/Put/seqid=0 2024-11-12T12:25:03,560 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/C of cfcbe6d6503fe59cbb8c8cb9085531ad into e80590448c64411da4d0ee52e1eb1441(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:03,560 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:03,560 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/C, priority=12, startTime=1731414303423; duration=0sec 2024-11-12T12:25:03,560 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:03,560 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:C 2024-11-12T12:25:03,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742005_1181 (size=39749) 2024-11-12T12:25:03,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414363651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414363651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414363651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414363651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414363651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414363862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414363862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414363862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414363862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414363862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:03,903 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/7f0be1ad39c6455292ad8e471baa5ba6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/7f0be1ad39c6455292ad8e471baa5ba6 2024-11-12T12:25:03,909 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/A of cfcbe6d6503fe59cbb8c8cb9085531ad into 7f0be1ad39c6455292ad8e471baa5ba6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:03,910 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:03,910 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/A, priority=12, startTime=1731414303423; duration=0sec 2024-11-12T12:25:03,912 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:03,912 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:A 2024-11-12T12:25:03,979 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/66e5878b13994e49bcd5c7b342cdf079 2024-11-12T12:25:04,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/634c71c43f41473ba35da6cbca43b0c8 is 50, key is test_row_0/B:col10/1731414303494/Put/seqid=0 2024-11-12T12:25:04,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742006_1182 (size=12151) 2024-11-12T12:25:04,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414364172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414364172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414364172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414364172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414364172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/634c71c43f41473ba35da6cbca43b0c8 2024-11-12T12:25:04,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/8dd461f3f13349aa9babde471b081fd8 is 50, key is test_row_0/C:col10/1731414303494/Put/seqid=0 2024-11-12T12:25:04,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742007_1183 (size=12151) 2024-11-12T12:25:04,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414364682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414364682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414364682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414364682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414364682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:04,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/8dd461f3f13349aa9babde471b081fd8 2024-11-12T12:25:04,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/66e5878b13994e49bcd5c7b342cdf079 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/66e5878b13994e49bcd5c7b342cdf079 2024-11-12T12:25:04,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/66e5878b13994e49bcd5c7b342cdf079, entries=200, sequenceid=212, filesize=38.8 K 2024-11-12T12:25:04,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/634c71c43f41473ba35da6cbca43b0c8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/634c71c43f41473ba35da6cbca43b0c8 2024-11-12T12:25:04,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/634c71c43f41473ba35da6cbca43b0c8, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T12:25:04,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/8dd461f3f13349aa9babde471b081fd8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/8dd461f3f13349aa9babde471b081fd8 2024-11-12T12:25:04,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/8dd461f3f13349aa9babde471b081fd8, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T12:25:04,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1431ms, sequenceid=212, compaction requested=false 2024-11-12T12:25:04,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:05,702 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:05,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:25:05,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414365711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:05,716 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in cfcbe6d6503fe59cbb8c8cb9085531ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:05,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e25acae1866d4a2698d09d87bb5e3b26_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414305703/Put/seqid=0 2024-11-12T12:25:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414365715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414365715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414365716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414365717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:05,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:05,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742008_1184 (size=22268) 2024-11-12T12:25:05,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414365831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:05,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414365831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:05,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414365831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:05,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414365831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414366041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414366042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414366042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414366042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:06,148 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e25acae1866d4a2698d09d87bb5e3b26_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e25acae1866d4a2698d09d87bb5e3b26_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:06,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2641f44b43154a0593285724a5a57f18, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:06,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2641f44b43154a0593285724a5a57f18 is 175, key is test_row_0/A:col10/1731414305703/Put/seqid=0 2024-11-12T12:25:06,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742009_1185 (size=65673) 2024-11-12T12:25:06,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414366351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414366352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414366352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414366352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T12:25:06,432 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:06,432 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-12T12:25:06,434 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T12:25:06,434 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:06,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T12:25:06,559 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2641f44b43154a0593285724a5a57f18 2024-11-12T12:25:06,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/17685fb455ac483ba1150c2691642204 is 50, key is test_row_0/B:col10/1731414305703/Put/seqid=0 2024-11-12T12:25:06,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T12:25:06,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:06,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:06,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:06,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:06,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:06,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742010_1186 (size=12151) 2024-11-12T12:25:06,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T12:25:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:06,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:06,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:06,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T12:25:06,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414366861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414366862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414366862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414366862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:06,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T12:25:06,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:06,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:06,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:06,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:06,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:06,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/17685fb455ac483ba1150c2691642204 2024-11-12T12:25:07,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/07722c880fcc464c8424e8e0de8ec4ec is 50, key is test_row_0/C:col10/1731414305703/Put/seqid=0 2024-11-12T12:25:07,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742011_1187 (size=12151) 2024-11-12T12:25:07,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/07722c880fcc464c8424e8e0de8ec4ec 2024-11-12T12:25:07,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/2641f44b43154a0593285724a5a57f18 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2641f44b43154a0593285724a5a57f18 2024-11-12T12:25:07,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2641f44b43154a0593285724a5a57f18, entries=350, sequenceid=241, filesize=64.1 K 2024-11-12T12:25:07,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/17685fb455ac483ba1150c2691642204 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/17685fb455ac483ba1150c2691642204 2024-11-12T12:25:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/17685fb455ac483ba1150c2691642204, entries=150, sequenceid=241, filesize=11.9 K 2024-11-12T12:25:07,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/07722c880fcc464c8424e8e0de8ec4ec as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/07722c880fcc464c8424e8e0de8ec4ec 2024-11-12T12:25:07,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/07722c880fcc464c8424e8e0de8ec4ec, entries=150, sequenceid=241, filesize=11.9 K 2024-11-12T12:25:07,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1339ms, sequenceid=241, compaction requested=true 2024-11-12T12:25:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:07,042 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:07,042 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:07,043 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:07,043 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:07,043 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/A is initiating minor compaction (all files) 2024-11-12T12:25:07,043 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/B is initiating minor compaction (all files) 2024-11-12T12:25:07,043 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/A in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:07,043 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/B in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:07,043 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/cb6851bc22e54e88a91d89ba5fb69b7a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/634c71c43f41473ba35da6cbca43b0c8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/17685fb455ac483ba1150c2691642204] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=36.0 K 2024-11-12T12:25:07,043 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/7f0be1ad39c6455292ad8e471baa5ba6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/66e5878b13994e49bcd5c7b342cdf079, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2641f44b43154a0593285724a5a57f18] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=133.8 K 2024-11-12T12:25:07,043 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:07,043 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/7f0be1ad39c6455292ad8e471baa5ba6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/66e5878b13994e49bcd5c7b342cdf079, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2641f44b43154a0593285724a5a57f18] 2024-11-12T12:25:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T12:25:07,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:07,045 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting cb6851bc22e54e88a91d89ba5fb69b7a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731414301646 2024-11-12T12:25:07,045 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f0be1ad39c6455292ad8e471baa5ba6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731414301646 2024-11-12T12:25:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,045 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-12T12:25:07,046 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66e5878b13994e49bcd5c7b342cdf079, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731414302824 2024-11-12T12:25:07,046 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 634c71c43f41473ba35da6cbca43b0c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731414302824 2024-11-12T12:25:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,047 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2641f44b43154a0593285724a5a57f18, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731414303516 2024-11-12T12:25:07,047 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 17685fb455ac483ba1150c2691642204, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731414303516 2024-11-12T12:25:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T12:25:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112863a645f010a438c87ee0b234a1366b5_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414305715/Put/seqid=0 2024-11-12T12:25:07,064 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:07,064 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#B#compaction#157 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:07,065 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/1cb27244c63a491bb7863ddb5d23a30d is 50, key is test_row_0/B:col10/1731414305703/Put/seqid=0 2024-11-12T12:25:07,073 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112050908f994124c4d9e86782d608de1c3_cfcbe6d6503fe59cbb8c8cb9085531ad store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,079 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112050908f994124c4d9e86782d608de1c3_cfcbe6d6503fe59cbb8c8cb9085531ad, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:07,080 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112050908f994124c4d9e86782d608de1c3_cfcbe6d6503fe59cbb8c8cb9085531ad because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742012_1188 (size=9814) 2024-11-12T12:25:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,104 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112863a645f010a438c87ee0b234a1366b5_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112863a645f010a438c87ee0b234a1366b5_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/a8f7954192e944efbdf44aaa8c5dedcd, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/a8f7954192e944efbdf44aaa8c5dedcd is 175, key is test_row_0/A:col10/1731414305715/Put/seqid=0 2024-11-12T12:25:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742013_1189 (size=12697) 2024-11-12T12:25:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,121 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/1cb27244c63a491bb7863ddb5d23a30d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1cb27244c63a491bb7863ddb5d23a30d 2024-11-12T12:25:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,128 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/B of cfcbe6d6503fe59cbb8c8cb9085531ad into 1cb27244c63a491bb7863ddb5d23a30d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:07,128 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:07,128 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/B, priority=13, startTime=1731414307042; duration=0sec 2024-11-12T12:25:07,128 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:07,128 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:B 2024-11-12T12:25:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,128 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:07,130 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:07,130 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/C is initiating minor compaction (all files) 2024-11-12T12:25:07,130 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/C in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:07,130 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e80590448c64411da4d0ee52e1eb1441, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/8dd461f3f13349aa9babde471b081fd8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/07722c880fcc464c8424e8e0de8ec4ec] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=36.0 K 2024-11-12T12:25:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,131 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting e80590448c64411da4d0ee52e1eb1441, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731414301646 2024-11-12T12:25:07,132 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dd461f3f13349aa9babde471b081fd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731414302824 2024-11-12T12:25:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,133 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 07722c880fcc464c8424e8e0de8ec4ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731414303516 2024-11-12T12:25:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742014_1190 (size=4469) 2024-11-12T12:25:07,163 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#A#compaction#158 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,163 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/647da5f3e5134554b18b74144095302b is 175, key is test_row_0/A:col10/1731414305703/Put/seqid=0 2024-11-12T12:25:07,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742015_1191 (size=22461) 2024-11-12T12:25:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,180 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#C#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,180 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/3eb539a559034870849e725da5e7c8ff is 50, key is test_row_0/C:col10/1731414305703/Put/seqid=0 2024-11-12T12:25:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742016_1192 (size=31651) 2024-11-12T12:25:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,197 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/647da5f3e5134554b18b74144095302b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/647da5f3e5134554b18b74144095302b 2024-11-12T12:25:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,206 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/A of cfcbe6d6503fe59cbb8c8cb9085531ad into 647da5f3e5134554b18b74144095302b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:07,206 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:07,206 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/A, priority=13, startTime=1731414307042; duration=0sec 2024-11-12T12:25:07,206 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:07,206 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:A 2024-11-12T12:25:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742017_1193 (size=12697) 2024-11-12T12:25:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,568 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/a8f7954192e944efbdf44aaa8c5dedcd 2024-11-12T12:25:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T12:25:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/ba9fc5897430422c82eadb178ee02db1 is 50, key is test_row_0/B:col10/1731414305715/Put/seqid=0 2024-11-12T12:25:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742018_1194 (size=9757) 2024-11-12T12:25:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,619 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/3eb539a559034870849e725da5e7c8ff as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/3eb539a559034870849e725da5e7c8ff 2024-11-12T12:25:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,621 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/ba9fc5897430422c82eadb178ee02db1 2024-11-12T12:25:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,641 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/C of cfcbe6d6503fe59cbb8c8cb9085531ad into 3eb539a559034870849e725da5e7c8ff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:07,641 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:07,641 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/C, priority=13, startTime=1731414307042; duration=0sec 2024-11-12T12:25:07,641 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:07,641 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:C 2024-11-12T12:25:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/2a1cfe4df2234757a9615a49dc33ba30 is 50, key is test_row_0/C:col10/1731414305715/Put/seqid=0 2024-11-12T12:25:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742019_1195 (size=9757) 2024-11-12T12:25:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/2a1cfe4df2234757a9615a49dc33ba30 2024-11-12T12:25:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/a8f7954192e944efbdf44aaa8c5dedcd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a8f7954192e944efbdf44aaa8c5dedcd 2024-11-12T12:25:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,693 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a8f7954192e944efbdf44aaa8c5dedcd, entries=100, sequenceid=251, filesize=21.9 K 2024-11-12T12:25:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/ba9fc5897430422c82eadb178ee02db1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ba9fc5897430422c82eadb178ee02db1 2024-11-12T12:25:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,700 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ba9fc5897430422c82eadb178ee02db1, entries=100, sequenceid=251, filesize=9.5 K 2024-11-12T12:25:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/2a1cfe4df2234757a9615a49dc33ba30 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/2a1cfe4df2234757a9615a49dc33ba30 2024-11-12T12:25:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,707 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/2a1cfe4df2234757a9615a49dc33ba30, entries=100, sequenceid=251, filesize=9.5 K 2024-11-12T12:25:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,709 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for cfcbe6d6503fe59cbb8c8cb9085531ad in 664ms, sequenceid=251, compaction requested=false 2024-11-12T12:25:07,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:07,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:07,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-12T12:25:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-12T12:25:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-12T12:25:07,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2770 sec 2024-11-12T12:25:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.2830 sec 2024-11-12T12:25:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:07,904 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:07,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:25:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:07,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ca6f5d3f242b4cb0b6defafc4e3e1152_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414307889/Put/seqid=0 2024-11-12T12:25:07,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:07,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414367942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:07,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:07,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414367948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:07,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414367952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414367952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414367953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:07,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742021_1197 (size=27698) 2024-11-12T12:25:08,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414368061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414368061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414368061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414368061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414368061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414368271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414368271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414368272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414368272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414368272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:08,366 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ca6f5d3f242b4cb0b6defafc4e3e1152_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ca6f5d3f242b4cb0b6defafc4e3e1152_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:08,367 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/a66820fa3aa94749906baed3d55a64c4, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:08,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/a66820fa3aa94749906baed3d55a64c4 is 175, key is test_row_0/A:col10/1731414307889/Put/seqid=0 2024-11-12T12:25:08,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742020_1196 (size=83485) 2024-11-12T12:25:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T12:25:08,581 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:08,582 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:08,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414368582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-12T12:25:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T12:25:08,584 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:08,585 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:08,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:08,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414368591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414368591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414368592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414368592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T12:25:08,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T12:25:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:08,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:08,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:08,770 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/a66820fa3aa94749906baed3d55a64c4 2024-11-12T12:25:08,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/b8032d7c4a344136a96a57f8c9ced710 is 50, key is test_row_0/B:col10/1731414307889/Put/seqid=0 2024-11-12T12:25:08,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742022_1198 (size=12301) 2024-11-12T12:25:08,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T12:25:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:08,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T12:25:09,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T12:25:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:09,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:09,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414369091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:09,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414369102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:09,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414369102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:09,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414369102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:09,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414369102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:09,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/b8032d7c4a344136a96a57f8c9ced710 2024-11-12T12:25:09,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/674350a53aea41c7a361c07f0dfda4fb is 50, key is test_row_0/C:col10/1731414307889/Put/seqid=0 2024-11-12T12:25:09,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742023_1199 (size=12301) 2024-11-12T12:25:09,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/674350a53aea41c7a361c07f0dfda4fb 2024-11-12T12:25:09,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T12:25:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:09,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:09,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/a66820fa3aa94749906baed3d55a64c4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a66820fa3aa94749906baed3d55a64c4 2024-11-12T12:25:09,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a66820fa3aa94749906baed3d55a64c4, entries=450, sequenceid=266, filesize=81.5 K 2024-11-12T12:25:09,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/b8032d7c4a344136a96a57f8c9ced710 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/b8032d7c4a344136a96a57f8c9ced710 2024-11-12T12:25:09,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/b8032d7c4a344136a96a57f8c9ced710, entries=150, sequenceid=266, filesize=12.0 K 2024-11-12T12:25:09,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/674350a53aea41c7a361c07f0dfda4fb as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/674350a53aea41c7a361c07f0dfda4fb 2024-11-12T12:25:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T12:25:09,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/674350a53aea41c7a361c07f0dfda4fb, entries=150, sequenceid=266, filesize=12.0 K 2024-11-12T12:25:09,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1310ms, sequenceid=266, compaction requested=true 2024-11-12T12:25:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:09,214 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:09,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:09,214 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:09,215 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:09,215 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:09,215 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/A is initiating minor compaction (all files) 2024-11-12T12:25:09,215 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/B is initiating minor compaction (all files) 2024-11-12T12:25:09,215 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/A in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:09,215 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/B in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:09,215 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/647da5f3e5134554b18b74144095302b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a8f7954192e944efbdf44aaa8c5dedcd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a66820fa3aa94749906baed3d55a64c4] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=134.4 K 2024-11-12T12:25:09,215 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1cb27244c63a491bb7863ddb5d23a30d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ba9fc5897430422c82eadb178ee02db1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/b8032d7c4a344136a96a57f8c9ced710] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=33.9 K 2024-11-12T12:25:09,215 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:09,215 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/647da5f3e5134554b18b74144095302b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a8f7954192e944efbdf44aaa8c5dedcd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a66820fa3aa94749906baed3d55a64c4] 2024-11-12T12:25:09,216 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647da5f3e5134554b18b74144095302b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731414303516 2024-11-12T12:25:09,216 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb27244c63a491bb7863ddb5d23a30d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731414303516 2024-11-12T12:25:09,216 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9fc5897430422c82eadb178ee02db1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731414305713 2024-11-12T12:25:09,216 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8f7954192e944efbdf44aaa8c5dedcd, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731414305713 2024-11-12T12:25:09,216 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting b8032d7c4a344136a96a57f8c9ced710, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731414307889 2024-11-12T12:25:09,216 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a66820fa3aa94749906baed3d55a64c4, keycount=450, bloomtype=ROW, size=81.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731414307785 2024-11-12T12:25:09,224 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:09,225 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a9aa079f4d68473f8c44f92f57b6bb11 is 50, key is test_row_0/B:col10/1731414307889/Put/seqid=0 2024-11-12T12:25:09,235 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:09,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742024_1200 (size=12949) 2024-11-12T12:25:09,240 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411125fd6020359064d15a9d20b4171456102_cfcbe6d6503fe59cbb8c8cb9085531ad store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:09,243 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411125fd6020359064d15a9d20b4171456102_cfcbe6d6503fe59cbb8c8cb9085531ad, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:09,243 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125fd6020359064d15a9d20b4171456102_cfcbe6d6503fe59cbb8c8cb9085531ad because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:09,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742025_1201 (size=4469) 2024-11-12T12:25:09,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T12:25:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:09,351 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-12T12:25:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112db87de7fefd340be991875382c39d252_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414307948/Put/seqid=0 2024-11-12T12:25:09,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742026_1202 (size=12454) 2024-11-12T12:25:09,643 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/a9aa079f4d68473f8c44f92f57b6bb11 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a9aa079f4d68473f8c44f92f57b6bb11 2024-11-12T12:25:09,650 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:09,650 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/B of cfcbe6d6503fe59cbb8c8cb9085531ad into a9aa079f4d68473f8c44f92f57b6bb11(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:09,650 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:09,650 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/69f5cebcecdf4fbd929453f5a83ad8f4 is 175, key is test_row_0/A:col10/1731414307889/Put/seqid=0 2024-11-12T12:25:09,650 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/B, priority=13, startTime=1731414309214; duration=0sec 2024-11-12T12:25:09,650 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:09,650 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:B 2024-11-12T12:25:09,650 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:09,653 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:09,653 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/C is initiating minor compaction (all files) 2024-11-12T12:25:09,653 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/C in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:09,653 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/3eb539a559034870849e725da5e7c8ff, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/2a1cfe4df2234757a9615a49dc33ba30, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/674350a53aea41c7a361c07f0dfda4fb] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=33.9 K 2024-11-12T12:25:09,654 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb539a559034870849e725da5e7c8ff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731414303516 2024-11-12T12:25:09,654 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1cfe4df2234757a9615a49dc33ba30, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731414305713 2024-11-12T12:25:09,654 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 674350a53aea41c7a361c07f0dfda4fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731414307889 2024-11-12T12:25:09,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742027_1203 (size=31903) 2024-11-12T12:25:09,674 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:09,674 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/0d565ed2a8cd4b7b8233d97e7a4f8c93 is 50, key is test_row_0/C:col10/1731414307889/Put/seqid=0 2024-11-12T12:25:09,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742028_1204 (size=12949) 2024-11-12T12:25:09,708 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/0d565ed2a8cd4b7b8233d97e7a4f8c93 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/0d565ed2a8cd4b7b8233d97e7a4f8c93 2024-11-12T12:25:09,714 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/C of cfcbe6d6503fe59cbb8c8cb9085531ad into 0d565ed2a8cd4b7b8233d97e7a4f8c93(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:09,714 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:09,714 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/C, priority=13, startTime=1731414309214; duration=0sec 2024-11-12T12:25:09,715 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:09,715 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:C 2024-11-12T12:25:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T12:25:09,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:09,767 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112db87de7fefd340be991875382c39d252_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112db87de7fefd340be991875382c39d252_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:09,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/1ea87c7f2a064c7f972fd8b0b140af54, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:09,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/1ea87c7f2a064c7f972fd8b0b140af54 is 175, key is test_row_0/A:col10/1731414307948/Put/seqid=0 2024-11-12T12:25:09,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742029_1205 (size=31255) 2024-11-12T12:25:09,782 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/1ea87c7f2a064c7f972fd8b0b140af54 2024-11-12T12:25:09,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/73fc71048c2b4eb7a011020e7253656e is 50, key is test_row_0/B:col10/1731414307948/Put/seqid=0 2024-11-12T12:25:09,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742030_1206 (size=12301) 2024-11-12T12:25:09,819 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/73fc71048c2b4eb7a011020e7253656e 2024-11-12T12:25:09,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/40bf155af4064ba18c458aa2fc0d0bc4 is 50, key is test_row_0/C:col10/1731414307948/Put/seqid=0 2024-11-12T12:25:09,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742031_1207 (size=12301) 2024-11-12T12:25:10,073 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/69f5cebcecdf4fbd929453f5a83ad8f4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/69f5cebcecdf4fbd929453f5a83ad8f4 2024-11-12T12:25:10,077 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/A of cfcbe6d6503fe59cbb8c8cb9085531ad into 69f5cebcecdf4fbd929453f5a83ad8f4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:10,077 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:10,077 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/A, priority=13, startTime=1731414309214; duration=0sec 2024-11-12T12:25:10,078 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:10,078 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:A 2024-11-12T12:25:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:10,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:10,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414370117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414370118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414370121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414370121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414370121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,122 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in cfcbe6d6503fe59cbb8c8cb9085531ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,123 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in cfcbe6d6503fe59cbb8c8cb9085531ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,123 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in cfcbe6d6503fe59cbb8c8cb9085531ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414370221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414370221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,243 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/40bf155af4064ba18c458aa2fc0d0bc4 2024-11-12T12:25:10,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/1ea87c7f2a064c7f972fd8b0b140af54 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/1ea87c7f2a064c7f972fd8b0b140af54 2024-11-12T12:25:10,251 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/1ea87c7f2a064c7f972fd8b0b140af54, entries=150, sequenceid=292, filesize=30.5 K 2024-11-12T12:25:10,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/73fc71048c2b4eb7a011020e7253656e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/73fc71048c2b4eb7a011020e7253656e 2024-11-12T12:25:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,255 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/73fc71048c2b4eb7a011020e7253656e, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T12:25:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/40bf155af4064ba18c458aa2fc0d0bc4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40bf155af4064ba18c458aa2fc0d0bc4 2024-11-12T12:25:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,261 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40bf155af4064ba18c458aa2fc0d0bc4, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T12:25:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,263 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for cfcbe6d6503fe59cbb8c8cb9085531ad in 912ms, sequenceid=292, compaction requested=false 2024-11-12T12:25:10,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:10,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-12T12:25:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-12T12:25:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-12T12:25:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6790 sec 2024-11-12T12:25:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.6850 sec 2024-11-12T12:25:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:10,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:25:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121ed74194e01e42e09b14cbf36f4ef77a_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414310450/Put/seqid=0 2024-11-12T12:25:10,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742032_1208 (size=14994) 2024-11-12T12:25:10,479 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121ed74194e01e42e09b14cbf36f4ef77a_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121ed74194e01e42e09b14cbf36f4ef77a_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:10,487 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/d6c3008171124e3f921d91d499a44ae7, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:10,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/d6c3008171124e3f921d91d499a44ae7 is 175, key is test_row_0/A:col10/1731414310450/Put/seqid=0 2024-11-12T12:25:10,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742033_1209 (size=39949) 2024-11-12T12:25:10,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/d6c3008171124e3f921d91d499a44ae7 2024-11-12T12:25:10,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/0338245935704e82bf06738c402805b5 is 50, key is test_row_0/B:col10/1731414310450/Put/seqid=0 2024-11-12T12:25:10,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414370522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414370523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742034_1210 (size=12301) 2024-11-12T12:25:10,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/0338245935704e82bf06738c402805b5 2024-11-12T12:25:10,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/e3daddd36953482eb55fc591761ed5a3 is 50, key is test_row_0/C:col10/1731414310450/Put/seqid=0 2024-11-12T12:25:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742035_1211 (size=12301) 2024-11-12T12:25:10,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/e3daddd36953482eb55fc591761ed5a3 2024-11-12T12:25:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/d6c3008171124e3f921d91d499a44ae7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/d6c3008171124e3f921d91d499a44ae7 2024-11-12T12:25:10,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/d6c3008171124e3f921d91d499a44ae7, entries=200, sequenceid=307, filesize=39.0 K 2024-11-12T12:25:10,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/0338245935704e82bf06738c402805b5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/0338245935704e82bf06738c402805b5 2024-11-12T12:25:10,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/0338245935704e82bf06738c402805b5, entries=150, sequenceid=307, filesize=12.0 K 2024-11-12T12:25:10,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/e3daddd36953482eb55fc591761ed5a3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e3daddd36953482eb55fc591761ed5a3 2024-11-12T12:25:10,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e3daddd36953482eb55fc591761ed5a3, entries=150, sequenceid=307, filesize=12.0 K 2024-11-12T12:25:10,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for cfcbe6d6503fe59cbb8c8cb9085531ad in 138ms, sequenceid=307, compaction requested=true 2024-11-12T12:25:10,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:10,588 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:10,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:10,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:10,588 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:10,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:10,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:10,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:10,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:10,589 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:10,589 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/A is initiating minor compaction (all files) 2024-11-12T12:25:10,589 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/A in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:10,589 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/69f5cebcecdf4fbd929453f5a83ad8f4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/1ea87c7f2a064c7f972fd8b0b140af54, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/d6c3008171124e3f921d91d499a44ae7] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=100.7 K 2024-11-12T12:25:10,589 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:10,590 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/69f5cebcecdf4fbd929453f5a83ad8f4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/1ea87c7f2a064c7f972fd8b0b140af54, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/d6c3008171124e3f921d91d499a44ae7] 2024-11-12T12:25:10,590 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:10,590 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 69f5cebcecdf4fbd929453f5a83ad8f4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731414307889 2024-11-12T12:25:10,590 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/B is initiating minor compaction (all files) 2024-11-12T12:25:10,590 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/B in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:10,590 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a9aa079f4d68473f8c44f92f57b6bb11, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/73fc71048c2b4eb7a011020e7253656e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/0338245935704e82bf06738c402805b5] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=36.7 K 2024-11-12T12:25:10,591 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea87c7f2a064c7f972fd8b0b140af54, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731414307946 2024-11-12T12:25:10,591 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9aa079f4d68473f8c44f92f57b6bb11, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731414307889 2024-11-12T12:25:10,591 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d6c3008171124e3f921d91d499a44ae7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731414310115 2024-11-12T12:25:10,592 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73fc71048c2b4eb7a011020e7253656e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731414307946 2024-11-12T12:25:10,592 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0338245935704e82bf06738c402805b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731414310115 2024-11-12T12:25:10,601 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:10,604 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112b8098a5e268545ca9453e091e1e695fe_cfcbe6d6503fe59cbb8c8cb9085531ad store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:10,607 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112b8098a5e268545ca9453e091e1e695fe_cfcbe6d6503fe59cbb8c8cb9085531ad, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:10,607 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b8098a5e268545ca9453e091e1e695fe_cfcbe6d6503fe59cbb8c8cb9085531ad because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:10,613 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:10,614 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/66bdec5529d24199b5e61972c05d7749 is 50, key is test_row_0/B:col10/1731414310450/Put/seqid=0 2024-11-12T12:25:10,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742036_1212 (size=4469) 2024-11-12T12:25:10,619 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#A#compaction#174 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:10,620 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/f0b7a0b227bc4e79be3b4b8115e04c5a is 175, key is test_row_0/A:col10/1731414310450/Put/seqid=0 2024-11-12T12:25:10,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742037_1213 (size=13051) 2024-11-12T12:25:10,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742038_1214 (size=32005) 2024-11-12T12:25:10,639 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/f0b7a0b227bc4e79be3b4b8115e04c5a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f0b7a0b227bc4e79be3b4b8115e04c5a 2024-11-12T12:25:10,644 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/A of cfcbe6d6503fe59cbb8c8cb9085531ad into f0b7a0b227bc4e79be3b4b8115e04c5a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:10,644 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:10,644 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/A, priority=13, startTime=1731414310588; duration=0sec 2024-11-12T12:25:10,644 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:10,644 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:A 2024-11-12T12:25:10,644 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:10,646 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:10,646 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/C is initiating minor compaction (all files) 2024-11-12T12:25:10,646 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/C in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:10,646 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/0d565ed2a8cd4b7b8233d97e7a4f8c93, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40bf155af4064ba18c458aa2fc0d0bc4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e3daddd36953482eb55fc591761ed5a3] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=36.7 K 2024-11-12T12:25:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:10,646 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:10,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-12T12:25:10,648 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d565ed2a8cd4b7b8233d97e7a4f8c93, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731414307889 2024-11-12T12:25:10,648 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 40bf155af4064ba18c458aa2fc0d0bc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731414307946 2024-11-12T12:25:10,649 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting e3daddd36953482eb55fc591761ed5a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731414310115 2024-11-12T12:25:10,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112752637d4c4ca4762b596bf337047bf13_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414310645/Put/seqid=0 2024-11-12T12:25:10,661 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#C#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:10,661 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/d146eeea704f4d92a7748fae002c9f88 is 50, key is test_row_0/C:col10/1731414310450/Put/seqid=0 2024-11-12T12:25:10,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414370664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742039_1215 (size=12454) 2024-11-12T12:25:10,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414370669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:10,678 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112752637d4c4ca4762b596bf337047bf13_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112752637d4c4ca4762b596bf337047bf13_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:10,679 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/9e428bafc2ca433a8bf0a2b4b436aa65, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:10,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742040_1216 (size=13051) 2024-11-12T12:25:10,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/9e428bafc2ca433a8bf0a2b4b436aa65 is 175, key is test_row_0/A:col10/1731414310645/Put/seqid=0 2024-11-12T12:25:10,688 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/d146eeea704f4d92a7748fae002c9f88 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/d146eeea704f4d92a7748fae002c9f88 2024-11-12T12:25:10,693 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/C of cfcbe6d6503fe59cbb8c8cb9085531ad into d146eeea704f4d92a7748fae002c9f88(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:10,693 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:10,693 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/C, priority=13, startTime=1731414310588; duration=0sec 2024-11-12T12:25:10,693 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:10,693 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:C 2024-11-12T12:25:10,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742041_1217 (size=31255) 2024-11-12T12:25:10,695 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/9e428bafc2ca433a8bf0a2b4b436aa65 2024-11-12T12:25:10,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/c2f8446b0a0b46b984bb51f21aa6afe8 is 50, key is test_row_0/B:col10/1731414310645/Put/seqid=0 2024-11-12T12:25:10,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742042_1218 (size=12301) 2024-11-12T12:25:10,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/c2f8446b0a0b46b984bb51f21aa6afe8 2024-11-12T12:25:10,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/a303e25046ae42508e5043d53bc66b2a is 50, key is test_row_0/C:col10/1731414310645/Put/seqid=0 2024-11-12T12:25:10,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742043_1219 (size=12301) 2024-11-12T12:25:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T12:25:10,731 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:10,731 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-12T12:25:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T12:25:10,733 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:10,735 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:10,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:10,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414370771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414370782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T12:25:10,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T12:25:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:10,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:10,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414370981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:10,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414370991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:10,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:11,032 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/66bdec5529d24199b5e61972c05d7749 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/66bdec5529d24199b5e61972c05d7749 2024-11-12T12:25:11,037 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/B of cfcbe6d6503fe59cbb8c8cb9085531ad into 66bdec5529d24199b5e61972c05d7749(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:11,037 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:11,037 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/B, priority=13, startTime=1731414310588; duration=0sec 2024-11-12T12:25:11,037 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:11,037 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:B 2024-11-12T12:25:11,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T12:25:11,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:11,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:11,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:11,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:11,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T12:25:11,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/a303e25046ae42508e5043d53bc66b2a 2024-11-12T12:25:11,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/9e428bafc2ca433a8bf0a2b4b436aa65 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/9e428bafc2ca433a8bf0a2b4b436aa65 2024-11-12T12:25:11,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/9e428bafc2ca433a8bf0a2b4b436aa65, entries=150, sequenceid=334, filesize=30.5 K 2024-11-12T12:25:11,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/c2f8446b0a0b46b984bb51f21aa6afe8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/c2f8446b0a0b46b984bb51f21aa6afe8 2024-11-12T12:25:11,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/c2f8446b0a0b46b984bb51f21aa6afe8, entries=150, sequenceid=334, filesize=12.0 K 2024-11-12T12:25:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/a303e25046ae42508e5043d53bc66b2a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/a303e25046ae42508e5043d53bc66b2a 2024-11-12T12:25:11,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/a303e25046ae42508e5043d53bc66b2a, entries=150, sequenceid=334, filesize=12.0 K 2024-11-12T12:25:11,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for cfcbe6d6503fe59cbb8c8cb9085531ad in 496ms, sequenceid=334, compaction requested=false 2024-11-12T12:25:11,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:11,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T12:25:11,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:11,193 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-12T12:25:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112581ddeb24183495397fc8981f7fcf48e_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414310648/Put/seqid=0 2024-11-12T12:25:11,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742044_1220 (size=12454) 2024-11-12T12:25:11,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:11,222 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112581ddeb24183495397fc8981f7fcf48e_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112581ddeb24183495397fc8981f7fcf48e_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:11,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/70730421c640425abd6792b1dda3aa98, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:11,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/70730421c640425abd6792b1dda3aa98 is 175, key is test_row_0/A:col10/1731414310648/Put/seqid=0 2024-11-12T12:25:11,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742045_1221 (size=31255) 2024-11-12T12:25:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:11,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. as already flushing 2024-11-12T12:25:11,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414371350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:11,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414371352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T12:25:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414371461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:11,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414371461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:11,627 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/70730421c640425abd6792b1dda3aa98 2024-11-12T12:25:11,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/6788faef73704d00aef53976296e5be0 is 50, key is test_row_0/B:col10/1731414310648/Put/seqid=0 2024-11-12T12:25:11,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742046_1222 (size=12301) 2024-11-12T12:25:11,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414371671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:11,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414371671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:11,829 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-12T12:25:11,829 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:11,829 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:11,829 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:11,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:11,830 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-12T12:25:11,830 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:11,830 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:11,830 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:11,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:11,831 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-12T12:25:11,831 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:11,831 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:11,831 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:11,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:11,832 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-12T12:25:11,832 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:11,832 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:11,832 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:11,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T12:25:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59056 deadline: 1731414371981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59072 deadline: 1731414371981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:12,041 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/6788faef73704d00aef53976296e5be0 2024-11-12T12:25:12,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/4e7ea872ca9b4a3c83f6cd9105230883 is 50, key is test_row_0/C:col10/1731414310648/Put/seqid=0 2024-11-12T12:25:12,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742047_1223 (size=12301) 2024-11-12T12:25:12,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:12,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58994 deadline: 1731414372132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59050 deadline: 1731414372132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:12,133 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in cfcbe6d6503fe59cbb8c8cb9085531ad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:12,133 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in cfcbe6d6503fe59cbb8c8cb9085531ad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:12,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59026 deadline: 1731414372141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:12,142 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in cfcbe6d6503fe59cbb8c8cb9085531ad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:12,453 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/4e7ea872ca9b4a3c83f6cd9105230883 2024-11-12T12:25:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/70730421c640425abd6792b1dda3aa98 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/70730421c640425abd6792b1dda3aa98 2024-11-12T12:25:12,471 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/70730421c640425abd6792b1dda3aa98, entries=150, sequenceid=346, filesize=30.5 K 2024-11-12T12:25:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/6788faef73704d00aef53976296e5be0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/6788faef73704d00aef53976296e5be0 2024-11-12T12:25:12,476 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/6788faef73704d00aef53976296e5be0, entries=150, sequenceid=346, filesize=12.0 K 2024-11-12T12:25:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/4e7ea872ca9b4a3c83f6cd9105230883 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4e7ea872ca9b4a3c83f6cd9105230883 2024-11-12T12:25:12,481 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4e7ea872ca9b4a3c83f6cd9105230883, entries=150, sequenceid=346, filesize=12.0 K 2024-11-12T12:25:12,482 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1289ms, sequenceid=346, compaction requested=true 2024-11-12T12:25:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-12T12:25:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-12T12:25:12,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-12T12:25:12,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7490 sec 2024-11-12T12:25:12,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.7550 sec 2024-11-12T12:25:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:12,492 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:12,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-12T12:25:12,492 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-12T12:25:12,492 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-12T12:25:12,492 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:12,492 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:12,492 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:12,492 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:12,492 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:12,492 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:12,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:12,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:12,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121a4333ac0b124bd9897fbd22345248e5_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414311345/Put/seqid=0 2024-11-12T12:25:12,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742048_1224 (size=12454) 2024-11-12T12:25:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T12:25:12,882 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:12,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:12,913 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121a4333ac0b124bd9897fbd22345248e5_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121a4333ac0b124bd9897fbd22345248e5_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:12,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/f496d541507746d8ae998a9db2d5a983, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:12,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/f496d541507746d8ae998a9db2d5a983 is 175, key is test_row_0/A:col10/1731414311345/Put/seqid=0 2024-11-12T12:25:12,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742049_1225 (size=31255) 2024-11-12T12:25:13,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/f496d541507746d8ae998a9db2d5a983 2024-11-12T12:25:13,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/e67ec910e97c4ed888a5c887cd3a6da4 is 50, key is test_row_0/B:col10/1731414311345/Put/seqid=0 2024-11-12T12:25:13,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742050_1226 (size=12301) 2024-11-12T12:25:13,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/e67ec910e97c4ed888a5c887cd3a6da4 2024-11-12T12:25:13,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/40b419f9fa4d47839fed6b4f05835ed7 is 50, key is test_row_0/C:col10/1731414311345/Put/seqid=0 2024-11-12T12:25:13,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742051_1227 (size=12301) 2024-11-12T12:25:14,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/40b419f9fa4d47839fed6b4f05835ed7 2024-11-12T12:25:14,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/f496d541507746d8ae998a9db2d5a983 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f496d541507746d8ae998a9db2d5a983 2024-11-12T12:25:14,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f496d541507746d8ae998a9db2d5a983, entries=150, sequenceid=374, filesize=30.5 K 2024-11-12T12:25:14,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/e67ec910e97c4ed888a5c887cd3a6da4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/e67ec910e97c4ed888a5c887cd3a6da4 2024-11-12T12:25:14,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/e67ec910e97c4ed888a5c887cd3a6da4, entries=150, sequenceid=374, filesize=12.0 K 2024-11-12T12:25:14,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/40b419f9fa4d47839fed6b4f05835ed7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40b419f9fa4d47839fed6b4f05835ed7 2024-11-12T12:25:14,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40b419f9fa4d47839fed6b4f05835ed7, entries=150, sequenceid=374, filesize=12.0 K 2024-11-12T12:25:14,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=0 B/0 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1691ms, sequenceid=374, compaction requested=true 2024-11-12T12:25:14,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:14,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:14,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:14,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:14,183 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:14,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:14,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfcbe6d6503fe59cbb8c8cb9085531ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:14,183 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:14,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:14,185 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:14,185 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:14,185 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/A is initiating minor compaction (all files) 2024-11-12T12:25:14,185 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/B is initiating minor compaction (all files) 2024-11-12T12:25:14,185 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/A in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:14,185 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/B in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:14,185 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/66bdec5529d24199b5e61972c05d7749, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/c2f8446b0a0b46b984bb51f21aa6afe8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/6788faef73704d00aef53976296e5be0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/e67ec910e97c4ed888a5c887cd3a6da4] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=48.8 K 2024-11-12T12:25:14,185 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f0b7a0b227bc4e79be3b4b8115e04c5a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/9e428bafc2ca433a8bf0a2b4b436aa65, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/70730421c640425abd6792b1dda3aa98, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f496d541507746d8ae998a9db2d5a983] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=122.8 K 2024-11-12T12:25:14,185 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:14,185 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f0b7a0b227bc4e79be3b4b8115e04c5a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/9e428bafc2ca433a8bf0a2b4b436aa65, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/70730421c640425abd6792b1dda3aa98, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f496d541507746d8ae998a9db2d5a983] 2024-11-12T12:25:14,185 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 66bdec5529d24199b5e61972c05d7749, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731414310115 2024-11-12T12:25:14,185 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0b7a0b227bc4e79be3b4b8115e04c5a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731414310115 2024-11-12T12:25:14,185 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting c2f8446b0a0b46b984bb51f21aa6afe8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731414310498 2024-11-12T12:25:14,186 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e428bafc2ca433a8bf0a2b4b436aa65, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731414310498 2024-11-12T12:25:14,186 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6788faef73704d00aef53976296e5be0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731414310648 2024-11-12T12:25:14,186 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70730421c640425abd6792b1dda3aa98, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731414310648 2024-11-12T12:25:14,186 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting e67ec910e97c4ed888a5c887cd3a6da4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731414311345 2024-11-12T12:25:14,186 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting f496d541507746d8ae998a9db2d5a983, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731414311345 2024-11-12T12:25:14,192 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:14,194 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112dd5a9bffc71e468e984ba531c52be250_cfcbe6d6503fe59cbb8c8cb9085531ad store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:14,195 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#B#compaction#186 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:14,196 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/beb0ed745db5456daeba1d26210e1cf4 is 50, key is test_row_0/B:col10/1731414311345/Put/seqid=0 2024-11-12T12:25:14,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742052_1228 (size=13187) 2024-11-12T12:25:14,232 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112dd5a9bffc71e468e984ba531c52be250_cfcbe6d6503fe59cbb8c8cb9085531ad, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:14,232 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dd5a9bffc71e468e984ba531c52be250_cfcbe6d6503fe59cbb8c8cb9085531ad because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:14,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742053_1229 (size=4469) 2024-11-12T12:25:14,608 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/beb0ed745db5456daeba1d26210e1cf4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/beb0ed745db5456daeba1d26210e1cf4 2024-11-12T12:25:14,615 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/B of cfcbe6d6503fe59cbb8c8cb9085531ad into beb0ed745db5456daeba1d26210e1cf4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:14,615 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:14,615 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/B, priority=12, startTime=1731414314183; duration=0sec 2024-11-12T12:25:14,615 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:14,615 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:B 2024-11-12T12:25:14,615 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:14,616 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:14,616 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): cfcbe6d6503fe59cbb8c8cb9085531ad/C is initiating minor compaction (all files) 2024-11-12T12:25:14,617 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfcbe6d6503fe59cbb8c8cb9085531ad/C in TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:14,617 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/d146eeea704f4d92a7748fae002c9f88, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/a303e25046ae42508e5043d53bc66b2a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4e7ea872ca9b4a3c83f6cd9105230883, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40b419f9fa4d47839fed6b4f05835ed7] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp, totalSize=48.8 K 2024-11-12T12:25:14,617 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d146eeea704f4d92a7748fae002c9f88, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731414310115 2024-11-12T12:25:14,617 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a303e25046ae42508e5043d53bc66b2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731414310498 2024-11-12T12:25:14,618 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e7ea872ca9b4a3c83f6cd9105230883, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731414310648 2024-11-12T12:25:14,618 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b419f9fa4d47839fed6b4f05835ed7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731414311345 2024-11-12T12:25:14,631 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#C#compaction#188 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:14,631 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/4424dadbc945462d8d1d518810b79519 is 50, key is test_row_0/C:col10/1731414311345/Put/seqid=0 2024-11-12T12:25:14,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742054_1230 (size=13187) 2024-11-12T12:25:14,637 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfcbe6d6503fe59cbb8c8cb9085531ad#A#compaction#187 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:14,638 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/6395a98cf48e4c898278a2986b3d7e4c is 175, key is test_row_0/A:col10/1731414311345/Put/seqid=0 2024-11-12T12:25:14,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742055_1231 (size=32141) 2024-11-12T12:25:15,041 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/4424dadbc945462d8d1d518810b79519 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4424dadbc945462d8d1d518810b79519 2024-11-12T12:25:15,045 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/6395a98cf48e4c898278a2986b3d7e4c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/6395a98cf48e4c898278a2986b3d7e4c 2024-11-12T12:25:15,045 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/C of cfcbe6d6503fe59cbb8c8cb9085531ad into 4424dadbc945462d8d1d518810b79519(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:15,045 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:15,046 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/C, priority=12, startTime=1731414314183; duration=0sec 2024-11-12T12:25:15,046 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:15,046 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:C 2024-11-12T12:25:15,049 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfcbe6d6503fe59cbb8c8cb9085531ad/A of cfcbe6d6503fe59cbb8c8cb9085531ad into 6395a98cf48e4c898278a2986b3d7e4c(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:15,050 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfcbe6d6503fe59cbb8c8cb9085531ad: 2024-11-12T12:25:15,050 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., storeName=cfcbe6d6503fe59cbb8c8cb9085531ad/A, priority=12, startTime=1731414314183; duration=0sec 2024-11-12T12:25:15,050 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:15,050 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfcbe6d6503fe59cbb8c8cb9085531ad:A 2024-11-12T12:25:16,046 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T12:25:16,154 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-12T12:25:16,155 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-12T12:25:16,155 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:16,155 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:16,155 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:16,155 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:16,155 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:16,155 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:16,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:16,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:16,164 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-12T12:25:16,164 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:16,164 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:16,164 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:16,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2785 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2740 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1245 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3735 rows 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1237 2024-11-12T12:25:16,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3708 rows 2024-11-12T12:25:16,165 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T12:25:16,165 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T12:25:16,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:16,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:16,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:16,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T12:25:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T12:25:16,170 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414316170"}]},"ts":"1731414316170"} 2024-11-12T12:25:16,172 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T12:25:16,172 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T12:25:16,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T12:25:16,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, UNASSIGN}] 2024-11-12T12:25:16,175 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, UNASSIGN 2024-11-12T12:25:16,176 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=cfcbe6d6503fe59cbb8c8cb9085531ad, regionState=CLOSING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:16,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, UNASSIGN because future has completed 2024-11-12T12:25:16,178 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T12:25:16,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:25:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T12:25:16,332 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:16,332 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T12:25:16,332 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing cfcbe6d6503fe59cbb8c8cb9085531ad, disabling compactions & flushes 2024-11-12T12:25:16,333 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:16,333 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:16,333 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. after waiting 0 ms 2024-11-12T12:25:16,333 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:16,333 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing cfcbe6d6503fe59cbb8c8cb9085531ad 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-12T12:25:16,346 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121f212dd0384948ab9092ade2c8223f6a_cfcbe6d6503fe59cbb8c8cb9085531ad is 50, key is test_row_0/A:col10/1731414316162/Put/seqid=0 2024-11-12T12:25:16,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742056_1232 (size=9914) 2024-11-12T12:25:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T12:25:16,751 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:16,761 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121f212dd0384948ab9092ade2c8223f6a_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121f212dd0384948ab9092ade2c8223f6a_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:16,762 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/71a1359d8fb144bfbd5dbf4264271eac, store: [table=TestAcidGuarantees family=A region=cfcbe6d6503fe59cbb8c8cb9085531ad] 2024-11-12T12:25:16,763 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/71a1359d8fb144bfbd5dbf4264271eac is 175, key is test_row_0/A:col10/1731414316162/Put/seqid=0 2024-11-12T12:25:16,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742057_1233 (size=22561) 2024-11-12T12:25:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T12:25:17,169 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/71a1359d8fb144bfbd5dbf4264271eac 2024-11-12T12:25:17,183 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/7d122d5656fc4c1a9c8a612eaa39924d is 50, key is test_row_0/B:col10/1731414316162/Put/seqid=0 2024-11-12T12:25:17,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742058_1234 (size=9857) 2024-11-12T12:25:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T12:25:17,588 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/7d122d5656fc4c1a9c8a612eaa39924d 2024-11-12T12:25:17,601 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/1bfe70377cb94c00b65cbefbaaee9f88 is 50, key is test_row_0/C:col10/1731414316162/Put/seqid=0 2024-11-12T12:25:17,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742059_1235 (size=9857) 2024-11-12T12:25:18,007 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/1bfe70377cb94c00b65cbefbaaee9f88 2024-11-12T12:25:18,017 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/A/71a1359d8fb144bfbd5dbf4264271eac as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/71a1359d8fb144bfbd5dbf4264271eac 2024-11-12T12:25:18,021 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/71a1359d8fb144bfbd5dbf4264271eac, entries=100, sequenceid=383, filesize=22.0 K 2024-11-12T12:25:18,022 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/B/7d122d5656fc4c1a9c8a612eaa39924d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/7d122d5656fc4c1a9c8a612eaa39924d 2024-11-12T12:25:18,027 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/7d122d5656fc4c1a9c8a612eaa39924d, entries=100, sequenceid=383, filesize=9.6 K 2024-11-12T12:25:18,027 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/.tmp/C/1bfe70377cb94c00b65cbefbaaee9f88 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/1bfe70377cb94c00b65cbefbaaee9f88 2024-11-12T12:25:18,031 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/1bfe70377cb94c00b65cbefbaaee9f88, entries=100, sequenceid=383, filesize=9.6 K 2024-11-12T12:25:18,032 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1699ms, sequenceid=383, compaction requested=false 2024-11-12T12:25:18,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/b2b493879858450c901370a90f513902, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/009728bf53864216b9856d368c97326c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8e62132ba4da46efb95b6249576300b2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/33daf090080c4026bddc7f410f9bb255, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8cdd2016375b4f5892c25a6ad12ec124, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2489ada82d6e48fd867072c658106569, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/3e741074918448d58d0852faab1841be, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/c44ca2b63bb344b1aa6fc2c07a911614, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2a2afc43e52743fdb273855e22dfa2bc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/e55006ad240f43428ded0ef55a7b0466, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/52274911d3ef4d8ea27fc226bc2337af, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8666501b8933482ca03828e5ebe5a524, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/7f0be1ad39c6455292ad8e471baa5ba6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/421694d1398843b69ebd569b2af91e23, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/66e5878b13994e49bcd5c7b342cdf079, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2641f44b43154a0593285724a5a57f18, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/647da5f3e5134554b18b74144095302b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a8f7954192e944efbdf44aaa8c5dedcd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a66820fa3aa94749906baed3d55a64c4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/69f5cebcecdf4fbd929453f5a83ad8f4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/1ea87c7f2a064c7f972fd8b0b140af54, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/d6c3008171124e3f921d91d499a44ae7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f0b7a0b227bc4e79be3b4b8115e04c5a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/9e428bafc2ca433a8bf0a2b4b436aa65, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/70730421c640425abd6792b1dda3aa98, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f496d541507746d8ae998a9db2d5a983] to archive 2024-11-12T12:25:18,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:25:18,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/b2b493879858450c901370a90f513902 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/b2b493879858450c901370a90f513902 2024-11-12T12:25:18,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/009728bf53864216b9856d368c97326c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/009728bf53864216b9856d368c97326c 2024-11-12T12:25:18,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8e62132ba4da46efb95b6249576300b2 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8e62132ba4da46efb95b6249576300b2 2024-11-12T12:25:18,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/33daf090080c4026bddc7f410f9bb255 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/33daf090080c4026bddc7f410f9bb255 2024-11-12T12:25:18,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8cdd2016375b4f5892c25a6ad12ec124 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8cdd2016375b4f5892c25a6ad12ec124 2024-11-12T12:25:18,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2489ada82d6e48fd867072c658106569 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2489ada82d6e48fd867072c658106569 2024-11-12T12:25:18,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/3e741074918448d58d0852faab1841be to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/3e741074918448d58d0852faab1841be 2024-11-12T12:25:18,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/c44ca2b63bb344b1aa6fc2c07a911614 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/c44ca2b63bb344b1aa6fc2c07a911614 2024-11-12T12:25:18,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2a2afc43e52743fdb273855e22dfa2bc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2a2afc43e52743fdb273855e22dfa2bc 2024-11-12T12:25:18,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/e55006ad240f43428ded0ef55a7b0466 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/e55006ad240f43428ded0ef55a7b0466 2024-11-12T12:25:18,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/52274911d3ef4d8ea27fc226bc2337af to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/52274911d3ef4d8ea27fc226bc2337af 2024-11-12T12:25:18,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8666501b8933482ca03828e5ebe5a524 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/8666501b8933482ca03828e5ebe5a524 2024-11-12T12:25:18,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/7f0be1ad39c6455292ad8e471baa5ba6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/7f0be1ad39c6455292ad8e471baa5ba6 2024-11-12T12:25:18,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/421694d1398843b69ebd569b2af91e23 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/421694d1398843b69ebd569b2af91e23 2024-11-12T12:25:18,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/66e5878b13994e49bcd5c7b342cdf079 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/66e5878b13994e49bcd5c7b342cdf079 2024-11-12T12:25:18,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2641f44b43154a0593285724a5a57f18 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/2641f44b43154a0593285724a5a57f18 2024-11-12T12:25:18,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/647da5f3e5134554b18b74144095302b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/647da5f3e5134554b18b74144095302b 2024-11-12T12:25:18,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a8f7954192e944efbdf44aaa8c5dedcd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a8f7954192e944efbdf44aaa8c5dedcd 2024-11-12T12:25:18,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a66820fa3aa94749906baed3d55a64c4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/a66820fa3aa94749906baed3d55a64c4 2024-11-12T12:25:18,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/69f5cebcecdf4fbd929453f5a83ad8f4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/69f5cebcecdf4fbd929453f5a83ad8f4 2024-11-12T12:25:18,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/1ea87c7f2a064c7f972fd8b0b140af54 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/1ea87c7f2a064c7f972fd8b0b140af54 2024-11-12T12:25:18,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/d6c3008171124e3f921d91d499a44ae7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/d6c3008171124e3f921d91d499a44ae7 2024-11-12T12:25:18,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f0b7a0b227bc4e79be3b4b8115e04c5a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f0b7a0b227bc4e79be3b4b8115e04c5a 2024-11-12T12:25:18,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/9e428bafc2ca433a8bf0a2b4b436aa65 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/9e428bafc2ca433a8bf0a2b4b436aa65 2024-11-12T12:25:18,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/70730421c640425abd6792b1dda3aa98 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/70730421c640425abd6792b1dda3aa98 2024-11-12T12:25:18,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f496d541507746d8ae998a9db2d5a983 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/f496d541507746d8ae998a9db2d5a983 2024-11-12T12:25:18,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/33fec82b8bdc4a968b497313b1bc7697, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/299517d162044c3b9f33a774627cdc7f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/88799061b50444a7a38b8a744380df83, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/540a98d0b0444033b93e186a90b781fa, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/71bb0aa4220049a1aa721514fb09b8d5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a7cf3edb9b174766b2c257a0616907ac, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/2d1536b2bf5a4d4eb654ecf7320338a3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/97f2d91ebb084540baa7dfc520a8a047, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/3fd8dd31201a47db92f5eda265f2aafd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a76e0b68e571445a9b96bfea44926179, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1e02330556ca42b7960361c4ae3e230e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ddbb57f1e8c44d6497b6b86f1145af7f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/cb6851bc22e54e88a91d89ba5fb69b7a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a2d4f05870c8492684d357ab9ce76e9c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/634c71c43f41473ba35da6cbca43b0c8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1cb27244c63a491bb7863ddb5d23a30d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/17685fb455ac483ba1150c2691642204, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ba9fc5897430422c82eadb178ee02db1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a9aa079f4d68473f8c44f92f57b6bb11, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/b8032d7c4a344136a96a57f8c9ced710, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/73fc71048c2b4eb7a011020e7253656e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/66bdec5529d24199b5e61972c05d7749, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/0338245935704e82bf06738c402805b5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/c2f8446b0a0b46b984bb51f21aa6afe8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/6788faef73704d00aef53976296e5be0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/e67ec910e97c4ed888a5c887cd3a6da4] to archive 2024-11-12T12:25:18,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:25:18,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/33fec82b8bdc4a968b497313b1bc7697 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/33fec82b8bdc4a968b497313b1bc7697 2024-11-12T12:25:18,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/299517d162044c3b9f33a774627cdc7f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/299517d162044c3b9f33a774627cdc7f 2024-11-12T12:25:18,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/88799061b50444a7a38b8a744380df83 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/88799061b50444a7a38b8a744380df83 2024-11-12T12:25:18,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/540a98d0b0444033b93e186a90b781fa to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/540a98d0b0444033b93e186a90b781fa 2024-11-12T12:25:18,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/71bb0aa4220049a1aa721514fb09b8d5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/71bb0aa4220049a1aa721514fb09b8d5 2024-11-12T12:25:18,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a7cf3edb9b174766b2c257a0616907ac to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a7cf3edb9b174766b2c257a0616907ac 2024-11-12T12:25:18,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/2d1536b2bf5a4d4eb654ecf7320338a3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/2d1536b2bf5a4d4eb654ecf7320338a3 2024-11-12T12:25:18,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/97f2d91ebb084540baa7dfc520a8a047 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/97f2d91ebb084540baa7dfc520a8a047 2024-11-12T12:25:18,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/3fd8dd31201a47db92f5eda265f2aafd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/3fd8dd31201a47db92f5eda265f2aafd 2024-11-12T12:25:18,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a76e0b68e571445a9b96bfea44926179 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a76e0b68e571445a9b96bfea44926179 2024-11-12T12:25:18,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1e02330556ca42b7960361c4ae3e230e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1e02330556ca42b7960361c4ae3e230e 2024-11-12T12:25:18,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ddbb57f1e8c44d6497b6b86f1145af7f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ddbb57f1e8c44d6497b6b86f1145af7f 2024-11-12T12:25:18,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/cb6851bc22e54e88a91d89ba5fb69b7a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/cb6851bc22e54e88a91d89ba5fb69b7a 2024-11-12T12:25:18,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a2d4f05870c8492684d357ab9ce76e9c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a2d4f05870c8492684d357ab9ce76e9c 2024-11-12T12:25:18,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/634c71c43f41473ba35da6cbca43b0c8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/634c71c43f41473ba35da6cbca43b0c8 2024-11-12T12:25:18,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1cb27244c63a491bb7863ddb5d23a30d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/1cb27244c63a491bb7863ddb5d23a30d 2024-11-12T12:25:18,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/17685fb455ac483ba1150c2691642204 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/17685fb455ac483ba1150c2691642204 2024-11-12T12:25:18,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ba9fc5897430422c82eadb178ee02db1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/ba9fc5897430422c82eadb178ee02db1 2024-11-12T12:25:18,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a9aa079f4d68473f8c44f92f57b6bb11 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/a9aa079f4d68473f8c44f92f57b6bb11 2024-11-12T12:25:18,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/b8032d7c4a344136a96a57f8c9ced710 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/b8032d7c4a344136a96a57f8c9ced710 2024-11-12T12:25:18,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/73fc71048c2b4eb7a011020e7253656e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/73fc71048c2b4eb7a011020e7253656e 2024-11-12T12:25:18,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/66bdec5529d24199b5e61972c05d7749 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/66bdec5529d24199b5e61972c05d7749 2024-11-12T12:25:18,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/0338245935704e82bf06738c402805b5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/0338245935704e82bf06738c402805b5 2024-11-12T12:25:18,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/c2f8446b0a0b46b984bb51f21aa6afe8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/c2f8446b0a0b46b984bb51f21aa6afe8 2024-11-12T12:25:18,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/6788faef73704d00aef53976296e5be0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/6788faef73704d00aef53976296e5be0 2024-11-12T12:25:18,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/e67ec910e97c4ed888a5c887cd3a6da4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/e67ec910e97c4ed888a5c887cd3a6da4 2024-11-12T12:25:18,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/50e2a1ef938841978f4347aa7ead70d6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/55333adca56849eb9f2baad1e6db7b2b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/619f8b3732794312bdc3a956f58018f2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e224a6cc622a4e928c0d9eeda5cc0932, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/6743a08f9e944d15b62a945d1f3294cd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/09c0ebee7aeb43f2bacc3b0f7ba5c5a4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4b073f3124794a5f8c2ddad746bc08ec, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/fc02f33b8949402ab5bbc0bc83725a43, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/28ea0fb4d9dd48c8ae5e6f30da8c4d2e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f04c92eb020a4db2bdf55fd9565878f2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/ac442f3b28594b49b44455b2a1c80a14, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f65b0292928048aeabbdfb661ccf32fd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e80590448c64411da4d0ee52e1eb1441, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/57e69806a5bd4b2fa321305141aba0d8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/8dd461f3f13349aa9babde471b081fd8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/3eb539a559034870849e725da5e7c8ff, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/07722c880fcc464c8424e8e0de8ec4ec, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/2a1cfe4df2234757a9615a49dc33ba30, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/0d565ed2a8cd4b7b8233d97e7a4f8c93, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/674350a53aea41c7a361c07f0dfda4fb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40bf155af4064ba18c458aa2fc0d0bc4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/d146eeea704f4d92a7748fae002c9f88, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e3daddd36953482eb55fc591761ed5a3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/a303e25046ae42508e5043d53bc66b2a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4e7ea872ca9b4a3c83f6cd9105230883, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40b419f9fa4d47839fed6b4f05835ed7] to archive 2024-11-12T12:25:18,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:25:18,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/50e2a1ef938841978f4347aa7ead70d6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/50e2a1ef938841978f4347aa7ead70d6 2024-11-12T12:25:18,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/55333adca56849eb9f2baad1e6db7b2b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/55333adca56849eb9f2baad1e6db7b2b 2024-11-12T12:25:18,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/619f8b3732794312bdc3a956f58018f2 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/619f8b3732794312bdc3a956f58018f2 2024-11-12T12:25:18,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e224a6cc622a4e928c0d9eeda5cc0932 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e224a6cc622a4e928c0d9eeda5cc0932 2024-11-12T12:25:18,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/6743a08f9e944d15b62a945d1f3294cd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/6743a08f9e944d15b62a945d1f3294cd 2024-11-12T12:25:18,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/09c0ebee7aeb43f2bacc3b0f7ba5c5a4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/09c0ebee7aeb43f2bacc3b0f7ba5c5a4 2024-11-12T12:25:18,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4b073f3124794a5f8c2ddad746bc08ec to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4b073f3124794a5f8c2ddad746bc08ec 2024-11-12T12:25:18,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/fc02f33b8949402ab5bbc0bc83725a43 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/fc02f33b8949402ab5bbc0bc83725a43 2024-11-12T12:25:18,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/28ea0fb4d9dd48c8ae5e6f30da8c4d2e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/28ea0fb4d9dd48c8ae5e6f30da8c4d2e 2024-11-12T12:25:18,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f04c92eb020a4db2bdf55fd9565878f2 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f04c92eb020a4db2bdf55fd9565878f2 2024-11-12T12:25:18,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/ac442f3b28594b49b44455b2a1c80a14 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/ac442f3b28594b49b44455b2a1c80a14 2024-11-12T12:25:18,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f65b0292928048aeabbdfb661ccf32fd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/f65b0292928048aeabbdfb661ccf32fd 2024-11-12T12:25:18,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e80590448c64411da4d0ee52e1eb1441 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e80590448c64411da4d0ee52e1eb1441 2024-11-12T12:25:18,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/57e69806a5bd4b2fa321305141aba0d8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/57e69806a5bd4b2fa321305141aba0d8 2024-11-12T12:25:18,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/8dd461f3f13349aa9babde471b081fd8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/8dd461f3f13349aa9babde471b081fd8 2024-11-12T12:25:18,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/3eb539a559034870849e725da5e7c8ff to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/3eb539a559034870849e725da5e7c8ff 2024-11-12T12:25:18,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/07722c880fcc464c8424e8e0de8ec4ec to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/07722c880fcc464c8424e8e0de8ec4ec 2024-11-12T12:25:18,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/2a1cfe4df2234757a9615a49dc33ba30 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/2a1cfe4df2234757a9615a49dc33ba30 2024-11-12T12:25:18,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/0d565ed2a8cd4b7b8233d97e7a4f8c93 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/0d565ed2a8cd4b7b8233d97e7a4f8c93 2024-11-12T12:25:18,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/674350a53aea41c7a361c07f0dfda4fb to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/674350a53aea41c7a361c07f0dfda4fb 2024-11-12T12:25:18,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40bf155af4064ba18c458aa2fc0d0bc4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40bf155af4064ba18c458aa2fc0d0bc4 2024-11-12T12:25:18,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/d146eeea704f4d92a7748fae002c9f88 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/d146eeea704f4d92a7748fae002c9f88 2024-11-12T12:25:18,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e3daddd36953482eb55fc591761ed5a3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/e3daddd36953482eb55fc591761ed5a3 2024-11-12T12:25:18,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/a303e25046ae42508e5043d53bc66b2a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/a303e25046ae42508e5043d53bc66b2a 2024-11-12T12:25:18,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4e7ea872ca9b4a3c83f6cd9105230883 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4e7ea872ca9b4a3c83f6cd9105230883 2024-11-12T12:25:18,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40b419f9fa4d47839fed6b4f05835ed7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/40b419f9fa4d47839fed6b4f05835ed7 2024-11-12T12:25:18,105 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/recovered.edits/386.seqid, newMaxSeqId=386, maxSeqId=4 2024-11-12T12:25:18,106 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. 2024-11-12T12:25:18,106 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for cfcbe6d6503fe59cbb8c8cb9085531ad: Waiting for close lock at 1731414316332Running coprocessor pre-close hooks at 1731414316332Disabling compacts and flushes for region at 1731414316332Disabling writes for close at 1731414316333 (+1 ms)Obtaining lock to block concurrent updates at 1731414316333Preparing flush snapshotting stores in cfcbe6d6503fe59cbb8c8cb9085531ad at 1731414316333Finished memstore snapshotting TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731414316334 (+1 ms)Flushing stores of TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad. at 1731414316335 (+1 ms)Flushing cfcbe6d6503fe59cbb8c8cb9085531ad/A: creating writer at 1731414316336 (+1 ms)Flushing mob file cfcbe6d6503fe59cbb8c8cb9085531ad/A: appending metadata at 1731414316346 (+10 ms)Flushing mob file cfcbe6d6503fe59cbb8c8cb9085531ad/A: closing flushed file at 1731414316346Flushing cfcbe6d6503fe59cbb8c8cb9085531ad/A: appending metadata at 1731414316762 (+416 ms)Flushing cfcbe6d6503fe59cbb8c8cb9085531ad/A: closing flushed file at 1731414316762Flushing cfcbe6d6503fe59cbb8c8cb9085531ad/B: creating writer at 1731414317180 (+418 ms)Flushing cfcbe6d6503fe59cbb8c8cb9085531ad/B: appending metadata at 1731414317182 (+2 ms)Flushing cfcbe6d6503fe59cbb8c8cb9085531ad/B: closing flushed file at 1731414317182Flushing cfcbe6d6503fe59cbb8c8cb9085531ad/C: creating writer at 1731414317598 (+416 ms)Flushing cfcbe6d6503fe59cbb8c8cb9085531ad/C: appending metadata at 1731414317601 (+3 ms)Flushing cfcbe6d6503fe59cbb8c8cb9085531ad/C: closing flushed file at 1731414317601Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a5bb12a: reopening flushed file at 1731414318016 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b357f4: reopening flushed file at 1731414318021 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7256878d: reopening flushed file at 1731414318027 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for cfcbe6d6503fe59cbb8c8cb9085531ad in 1699ms, sequenceid=383, compaction requested=false at 1731414318032 (+5 ms)Writing region close event to WAL at 1731414318102 (+70 ms)Running coprocessor post-close hooks at 1731414318105 (+3 ms)Closed at 1731414318106 (+1 ms) 2024-11-12T12:25:18,108 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,108 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=cfcbe6d6503fe59cbb8c8cb9085531ad, regionState=CLOSED 2024-11-12T12:25:18,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:25:18,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-12T12:25:18,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure cfcbe6d6503fe59cbb8c8cb9085531ad, server=4a91c05c96a4,38717,1731414257691 in 1.9320 sec 2024-11-12T12:25:18,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-12T12:25:18,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfcbe6d6503fe59cbb8c8cb9085531ad, UNASSIGN in 1.9370 sec 2024-11-12T12:25:18,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-12T12:25:18,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9400 sec 2024-11-12T12:25:18,128 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414318128"}]},"ts":"1731414318128"} 2024-11-12T12:25:18,129 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T12:25:18,129 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T12:25:18,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9640 sec 2024-11-12T12:25:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T12:25:18,321 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:18,322 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T12:25:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:18,323 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T12:25:18,324 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:18,326 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,329 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/recovered.edits] 2024-11-12T12:25:18,332 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/6395a98cf48e4c898278a2986b3d7e4c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/6395a98cf48e4c898278a2986b3d7e4c 2024-11-12T12:25:18,333 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/71a1359d8fb144bfbd5dbf4264271eac to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/A/71a1359d8fb144bfbd5dbf4264271eac 2024-11-12T12:25:18,335 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/7d122d5656fc4c1a9c8a612eaa39924d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/7d122d5656fc4c1a9c8a612eaa39924d 2024-11-12T12:25:18,336 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/beb0ed745db5456daeba1d26210e1cf4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/B/beb0ed745db5456daeba1d26210e1cf4 2024-11-12T12:25:18,338 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/1bfe70377cb94c00b65cbefbaaee9f88 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/1bfe70377cb94c00b65cbefbaaee9f88 2024-11-12T12:25:18,339 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4424dadbc945462d8d1d518810b79519 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/C/4424dadbc945462d8d1d518810b79519 2024-11-12T12:25:18,341 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/recovered.edits/386.seqid to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad/recovered.edits/386.seqid 2024-11-12T12:25:18,342 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,342 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T12:25:18,342 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T12:25:18,343 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T12:25:18,346 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121a4333ac0b124bd9897fbd22345248e5_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121a4333ac0b124bd9897fbd22345248e5_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,347 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121ed74194e01e42e09b14cbf36f4ef77a_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121ed74194e01e42e09b14cbf36f4ef77a_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,348 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121f212dd0384948ab9092ade2c8223f6a_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121f212dd0384948ab9092ade2c8223f6a_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,349 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111228e861f288d641f0a5d782fe5f232be4_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111228e861f288d641f0a5d782fe5f232be4_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,349 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111230198413dd6c470f8606cda4b39b039e_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111230198413dd6c470f8606cda4b39b039e_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,350 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111242498afcc10f44a4b6e8e449923d5f1f_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111242498afcc10f44a4b6e8e449923d5f1f_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,351 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111246033d5046514cb2a30cb58eba181351_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111246033d5046514cb2a30cb58eba181351_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,352 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112581ddeb24183495397fc8981f7fcf48e_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112581ddeb24183495397fc8981f7fcf48e_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,353 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111269124b2726f04932bcc19a2f461b16f5_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111269124b2726f04932bcc19a2f461b16f5_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,353 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112752637d4c4ca4762b596bf337047bf13_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112752637d4c4ca4762b596bf337047bf13_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,354 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127e13b55dd22f462bab705ce16f63467c_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127e13b55dd22f462bab705ce16f63467c_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,355 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127f7bdf7b60ca4d709f2bf2dfb91585a1_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127f7bdf7b60ca4d709f2bf2dfb91585a1_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,356 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112863a645f010a438c87ee0b234a1366b5_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112863a645f010a438c87ee0b234a1366b5_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,357 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bed0c27904354ea994c3cba45d381849_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bed0c27904354ea994c3cba45d381849_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,358 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ca6f5d3f242b4cb0b6defafc4e3e1152_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ca6f5d3f242b4cb0b6defafc4e3e1152_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,360 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d8ddce526b2a461dbd2c9ce178bb3b52_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d8ddce526b2a461dbd2c9ce178bb3b52_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,361 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112db87de7fefd340be991875382c39d252_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112db87de7fefd340be991875382c39d252_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,362 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112dec9c90414464a98ab2b5587c7c886a7_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112dec9c90414464a98ab2b5587c7c886a7_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,363 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e25acae1866d4a2698d09d87bb5e3b26_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e25acae1866d4a2698d09d87bb5e3b26_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,364 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e4be8f7312234714af95e4e31a9203fe_cfcbe6d6503fe59cbb8c8cb9085531ad to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e4be8f7312234714af95e4e31a9203fe_cfcbe6d6503fe59cbb8c8cb9085531ad 2024-11-12T12:25:18,365 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T12:25:18,367 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:18,369 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T12:25:18,371 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T12:25:18,372 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:18,372 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T12:25:18,372 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731414318372"}]},"ts":"9223372036854775807"} 2024-11-12T12:25:18,374 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T12:25:18,374 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cfcbe6d6503fe59cbb8c8cb9085531ad, NAME => 'TestAcidGuarantees,,1731414288523.cfcbe6d6503fe59cbb8c8cb9085531ad.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T12:25:18,374 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T12:25:18,374 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731414318374"}]},"ts":"9223372036854775807"} 2024-11-12T12:25:18,376 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T12:25:18,376 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:18,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-12T12:25:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T12:25:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T12:25:18,432 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:18,446 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1939194800_22 at /127.0.0.1:59098 [Waiting for operation #898] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1936992572_22 at /127.0.0.1:58924 [Waiting for operation #939] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1936992572_22 at /127.0.0.1:59102 [Waiting for operation #912] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1939194800_22 at /127.0.0.1:42852 [Waiting for operation #947] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=583 (was 496) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6963 (was 7217) 2024-11-12T12:25:18,458 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=583, ProcessCount=11, AvailableMemoryMB=6963 2024-11-12T12:25:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T12:25:18,460 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:25:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:18,462 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T12:25:18,463 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:18,463 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-12T12:25:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T12:25:18,464 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T12:25:18,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742060_1236 (size=959) 2024-11-12T12:25:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T12:25:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T12:25:18,874 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a 2024-11-12T12:25:18,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742061_1237 (size=53) 2024-11-12T12:25:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T12:25:19,284 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:25:19,284 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d1a5c014bc05d48b1ac89d4e1685996f, disabling compactions & flushes 2024-11-12T12:25:19,284 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:19,284 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:19,284 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. after waiting 0 ms 2024-11-12T12:25:19,284 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:19,285 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:19,285 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1a5c014bc05d48b1ac89d4e1685996f: Waiting for close lock at 1731414319284Disabling compacts and flushes for region at 1731414319284Disabling writes for close at 1731414319284Writing region close event to WAL at 1731414319284Closed at 1731414319284 2024-11-12T12:25:19,286 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T12:25:19,286 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731414319286"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731414319286"}]},"ts":"1731414319286"} 2024-11-12T12:25:19,288 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T12:25:19,289 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T12:25:19,289 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414319289"}]},"ts":"1731414319289"} 2024-11-12T12:25:19,291 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T12:25:19,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1a5c014bc05d48b1ac89d4e1685996f, ASSIGN}] 2024-11-12T12:25:19,293 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1a5c014bc05d48b1ac89d4e1685996f, ASSIGN 2024-11-12T12:25:19,294 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1a5c014bc05d48b1ac89d4e1685996f, ASSIGN; state=OFFLINE, location=4a91c05c96a4,38717,1731414257691; forceNewPlan=false, retain=false 2024-11-12T12:25:19,445 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=d1a5c014bc05d48b1ac89d4e1685996f, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:19,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1a5c014bc05d48b1ac89d4e1685996f, ASSIGN because future has completed 2024-11-12T12:25:19,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:25:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T12:25:19,610 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:19,610 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:25:19,611 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,611 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:25:19,611 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,612 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,614 INFO [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,615 INFO [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a5c014bc05d48b1ac89d4e1685996f columnFamilyName A 2024-11-12T12:25:19,615 DEBUG [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:19,616 INFO [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] regionserver.HStore(327): Store=d1a5c014bc05d48b1ac89d4e1685996f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:25:19,617 INFO [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,618 INFO [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a5c014bc05d48b1ac89d4e1685996f columnFamilyName B 2024-11-12T12:25:19,618 DEBUG [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:19,619 INFO [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] regionserver.HStore(327): Store=d1a5c014bc05d48b1ac89d4e1685996f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:25:19,619 INFO [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,620 INFO [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a5c014bc05d48b1ac89d4e1685996f columnFamilyName C 2024-11-12T12:25:19,621 DEBUG [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:19,621 INFO [StoreOpener-d1a5c014bc05d48b1ac89d4e1685996f-1 {}] regionserver.HStore(327): Store=d1a5c014bc05d48b1ac89d4e1685996f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:25:19,621 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,622 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,623 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,624 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,624 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,625 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:25:19,626 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,627 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T12:25:19,628 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened d1a5c014bc05d48b1ac89d4e1685996f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60548328, jitterRate=-0.09775960445404053}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:25:19,628 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:19,628 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for d1a5c014bc05d48b1ac89d4e1685996f: Running coprocessor pre-open hook at 1731414319612Writing region info on filesystem at 1731414319612Initializing all the Stores at 1731414319613 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414319613Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414319613Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414319613Cleaning up temporary data from old regions at 1731414319624 (+11 ms)Running coprocessor post-open hooks at 1731414319628 (+4 ms)Region opened successfully at 1731414319628 2024-11-12T12:25:19,629 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., pid=63, masterSystemTime=1731414319601 2024-11-12T12:25:19,631 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:19,631 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:19,631 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=d1a5c014bc05d48b1ac89d4e1685996f, regionState=OPEN, openSeqNum=2, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:19,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:25:19,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T12:25:19,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 in 186 msec 2024-11-12T12:25:19,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-12T12:25:19,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1a5c014bc05d48b1ac89d4e1685996f, ASSIGN in 344 msec 2024-11-12T12:25:19,637 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T12:25:19,637 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414319637"}]},"ts":"1731414319637"} 2024-11-12T12:25:19,638 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T12:25:19,639 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T12:25:19,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-12T12:25:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T12:25:20,622 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:20,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,624 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1e11c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,626 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38bdda9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,629 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T12:25:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a91c05c96a4,41939,1731414256942 2024-11-12T12:25:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@bde8437 2024-11-12T12:25:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T12:25:20,630 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T12:25:20,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,632 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14531d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f52d42f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb5b38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,636 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1603aa88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,638 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fafd5b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,639 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffacda1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@873b5f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293689ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,645 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5484f012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,646 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66dc42d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,648 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3508b2d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,649 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e1860f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,650 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4926fb98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,652 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216fa451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,654 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b97574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,655 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0ec31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,657 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6492c204, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,658 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f318eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:20,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:20,660 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc04334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:20,661 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f6e9a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:20,662 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,662 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,662 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,662 DEBUG [Thread-1118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,662 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,663 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:20,663 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,663 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,664 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,664 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-12T12:25:20,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T12:25:20,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,667 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,668 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,671 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:20,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:25:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:20,679 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:20,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:25:20,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/838d4726ea7a49f6b9d64366651f7c5e is 50, key is test_row_0/A:col10/1731414320679/Put/seqid=0 2024-11-12T12:25:20,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742062_1238 (size=12001) 2024-11-12T12:25:20,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414380709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414380709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414380713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414380713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414380715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T12:25:20,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414380821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414380821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414380821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414380822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T12:25:20,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:20,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:20,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:20,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:20,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:20,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414380831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:20,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T12:25:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:20,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T12:25:21,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414381031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414381032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414381032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414381032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414381041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/838d4726ea7a49f6b9d64366651f7c5e 2024-11-12T12:25:21,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T12:25:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4ca0752603284bb9bd1da2b987c6e834 is 50, key is test_row_0/B:col10/1731414320679/Put/seqid=0 2024-11-12T12:25:21,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742063_1239 (size=12001) 2024-11-12T12:25:21,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T12:25:21,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:21,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T12:25:21,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414381341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414381341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414381342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414381342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414381351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,390 INFO [AsyncFSWAL-0-hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData-prefix:4a91c05c96a4,41939,1731414256942 {}] wal.AbstractFSWAL(1368): Slow sync cost: 104 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33211,DS-3716a86f-e828-47a2-a25a-f34ffd346008,DISK]] 2024-11-12T12:25:21,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T12:25:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4ca0752603284bb9bd1da2b987c6e834 2024-11-12T12:25:21,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/ce0e3de6907f48bf92ad6276d3a6a1fc is 50, key is test_row_0/C:col10/1731414320679/Put/seqid=0 2024-11-12T12:25:21,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742064_1240 (size=12001) 2024-11-12T12:25:21,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T12:25:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T12:25:21,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T12:25:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:21,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:21,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414381851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414381851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414381851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414381851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414381861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:21,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/ce0e3de6907f48bf92ad6276d3a6a1fc 2024-11-12T12:25:21,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/838d4726ea7a49f6b9d64366651f7c5e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/838d4726ea7a49f6b9d64366651f7c5e 2024-11-12T12:25:21,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/838d4726ea7a49f6b9d64366651f7c5e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T12:25:21,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4ca0752603284bb9bd1da2b987c6e834 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4ca0752603284bb9bd1da2b987c6e834 2024-11-12T12:25:21,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4ca0752603284bb9bd1da2b987c6e834, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T12:25:21,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/ce0e3de6907f48bf92ad6276d3a6a1fc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ce0e3de6907f48bf92ad6276d3a6a1fc 2024-11-12T12:25:21,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ce0e3de6907f48bf92ad6276d3a6a1fc, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T12:25:21,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d1a5c014bc05d48b1ac89d4e1685996f in 1318ms, sequenceid=13, compaction requested=false 2024-11-12T12:25:21,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:22,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T12:25:22,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:22,005 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-12T12:25:22,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/0a5c760b1e7144d9b6ae260d4070f334 is 50, key is test_row_0/A:col10/1731414320713/Put/seqid=0 2024-11-12T12:25:22,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742065_1241 (size=12001) 2024-11-12T12:25:22,416 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/0a5c760b1e7144d9b6ae260d4070f334 2024-11-12T12:25:22,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/96b2828b84d34f0dbf1f4e9c7307af42 is 50, key is test_row_0/B:col10/1731414320713/Put/seqid=0 2024-11-12T12:25:22,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742066_1242 (size=12001) 2024-11-12T12:25:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T12:25:22,831 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/96b2828b84d34f0dbf1f4e9c7307af42 2024-11-12T12:25:22,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/12231e634602499282216bb17a4faf8b is 50, key is test_row_0/C:col10/1731414320713/Put/seqid=0 2024-11-12T12:25:22,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742067_1243 (size=12001) 2024-11-12T12:25:22,846 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/12231e634602499282216bb17a4faf8b 2024-11-12T12:25:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/0a5c760b1e7144d9b6ae260d4070f334 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5c760b1e7144d9b6ae260d4070f334 2024-11-12T12:25:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:22,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:22,863 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5c760b1e7144d9b6ae260d4070f334, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T12:25:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/96b2828b84d34f0dbf1f4e9c7307af42 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96b2828b84d34f0dbf1f4e9c7307af42 2024-11-12T12:25:22,875 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96b2828b84d34f0dbf1f4e9c7307af42, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T12:25:22,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/12231e634602499282216bb17a4faf8b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/12231e634602499282216bb17a4faf8b 2024-11-12T12:25:22,885 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/12231e634602499282216bb17a4faf8b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T12:25:22,886 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for d1a5c014bc05d48b1ac89d4e1685996f in 881ms, sequenceid=37, compaction requested=false 2024-11-12T12:25:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-12T12:25:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-12T12:25:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:22,888 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:22,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-12T12:25:22,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-12T12:25:22,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2170 sec 2024-11-12T12:25:22,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.2280 sec 2024-11-12T12:25:22,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/9fb6987ac63d446583c310156ea7ec5b is 50, key is test_row_0/A:col10/1731414322887/Put/seqid=0 2024-11-12T12:25:22,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742068_1244 (size=12001) 2024-11-12T12:25:22,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414382912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:22,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414382912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414382913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414382915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414382916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414383021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414383021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414383021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414383021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414383031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414383231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414383231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414383231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414383231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414383241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/9fb6987ac63d446583c310156ea7ec5b 2024-11-12T12:25:23,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/c2235face2db47718c96291cb534b594 is 50, key is test_row_0/B:col10/1731414322887/Put/seqid=0 2024-11-12T12:25:23,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742069_1245 (size=12001) 2024-11-12T12:25:23,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414383541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414383541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414383542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414383542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414383551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:23,637 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T12:25:23,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/c2235face2db47718c96291cb534b594 2024-11-12T12:25:23,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/917fcb2524fa412da821270930028913 is 50, key is test_row_0/C:col10/1731414322887/Put/seqid=0 2024-11-12T12:25:23,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742070_1246 (size=12001) 2024-11-12T12:25:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414384051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414384051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:24,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414384051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:24,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414384051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:24,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414384061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:24,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:24,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:24,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/917fcb2524fa412da821270930028913 2024-11-12T12:25:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/9fb6987ac63d446583c310156ea7ec5b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9fb6987ac63d446583c310156ea7ec5b 2024-11-12T12:25:24,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9fb6987ac63d446583c310156ea7ec5b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-12T12:25:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/c2235face2db47718c96291cb534b594 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/c2235face2db47718c96291cb534b594 2024-11-12T12:25:24,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/c2235face2db47718c96291cb534b594, entries=150, sequenceid=52, filesize=11.7 K 2024-11-12T12:25:24,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/917fcb2524fa412da821270930028913 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/917fcb2524fa412da821270930028913 2024-11-12T12:25:24,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/917fcb2524fa412da821270930028913, entries=150, sequenceid=52, filesize=11.7 K 2024-11-12T12:25:24,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d1a5c014bc05d48b1ac89d4e1685996f in 1267ms, sequenceid=52, compaction requested=true 2024-11-12T12:25:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:24,155 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:24,155 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:24,156 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:24,156 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:24,156 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/B is initiating minor compaction (all files) 2024-11-12T12:25:24,156 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/A is initiating minor compaction (all files) 2024-11-12T12:25:24,156 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/A in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:24,156 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/B in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:24,156 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/838d4726ea7a49f6b9d64366651f7c5e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5c760b1e7144d9b6ae260d4070f334, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9fb6987ac63d446583c310156ea7ec5b] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=35.2 K 2024-11-12T12:25:24,156 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4ca0752603284bb9bd1da2b987c6e834, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96b2828b84d34f0dbf1f4e9c7307af42, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/c2235face2db47718c96291cb534b594] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=35.2 K 2024-11-12T12:25:24,156 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca0752603284bb9bd1da2b987c6e834, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731414320675 2024-11-12T12:25:24,156 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 838d4726ea7a49f6b9d64366651f7c5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731414320675 2024-11-12T12:25:24,157 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b2828b84d34f0dbf1f4e9c7307af42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731414320706 2024-11-12T12:25:24,157 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a5c760b1e7144d9b6ae260d4070f334, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731414320706 2024-11-12T12:25:24,157 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting c2235face2db47718c96291cb534b594, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414322882 2024-11-12T12:25:24,157 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fb6987ac63d446583c310156ea7ec5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414322882 2024-11-12T12:25:24,165 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:24,165 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/2cbee7930239440299cb137a5af94beb is 50, key is test_row_0/B:col10/1731414322887/Put/seqid=0 2024-11-12T12:25:24,167 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#A#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:24,168 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/c9ef1107952d4142ae600bdcd3c76c3b is 50, key is test_row_0/A:col10/1731414322887/Put/seqid=0 2024-11-12T12:25:24,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742071_1247 (size=12104) 2024-11-12T12:25:24,182 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/2cbee7930239440299cb137a5af94beb as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2cbee7930239440299cb137a5af94beb 2024-11-12T12:25:24,187 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/B of d1a5c014bc05d48b1ac89d4e1685996f into 2cbee7930239440299cb137a5af94beb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:24,187 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:24,187 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/B, priority=13, startTime=1731414324155; duration=0sec 2024-11-12T12:25:24,187 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:24,187 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:B 2024-11-12T12:25:24,187 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:24,188 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:24,188 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/C is initiating minor compaction (all files) 2024-11-12T12:25:24,188 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/C in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:24,188 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ce0e3de6907f48bf92ad6276d3a6a1fc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/12231e634602499282216bb17a4faf8b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/917fcb2524fa412da821270930028913] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=35.2 K 2024-11-12T12:25:24,189 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ce0e3de6907f48bf92ad6276d3a6a1fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731414320675 2024-11-12T12:25:24,189 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 12231e634602499282216bb17a4faf8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731414320706 2024-11-12T12:25:24,190 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 917fcb2524fa412da821270930028913, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414322882 2024-11-12T12:25:24,197 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:24,197 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/88b9520ff40840038ef3197be90bed8f is 50, key is test_row_0/C:col10/1731414322887/Put/seqid=0 2024-11-12T12:25:24,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742072_1248 (size=12104) 2024-11-12T12:25:24,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742073_1249 (size=12104) 2024-11-12T12:25:24,207 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/c9ef1107952d4142ae600bdcd3c76c3b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/c9ef1107952d4142ae600bdcd3c76c3b 2024-11-12T12:25:24,212 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/A of d1a5c014bc05d48b1ac89d4e1685996f into c9ef1107952d4142ae600bdcd3c76c3b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:24,212 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:24,212 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/A, priority=13, startTime=1731414324155; duration=0sec 2024-11-12T12:25:24,212 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:24,213 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:A 2024-11-12T12:25:24,213 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/88b9520ff40840038ef3197be90bed8f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/88b9520ff40840038ef3197be90bed8f 2024-11-12T12:25:24,218 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/C of d1a5c014bc05d48b1ac89d4e1685996f into 88b9520ff40840038ef3197be90bed8f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:24,218 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:24,218 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/C, priority=13, startTime=1731414324155; duration=0sec 2024-11-12T12:25:24,218 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:24,218 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:C 2024-11-12T12:25:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T12:25:24,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:24,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-12T12:25:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T12:25:24,849 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:24,850 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:24,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T12:25:25,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T12:25:25,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:25,002 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-12T12:25:25,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/cd4a0a7fcad34a6a979526b0edb44172 is 50, key is test_row_0/A:col10/1731414322910/Put/seqid=0 2024-11-12T12:25:25,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742074_1250 (size=12001) 2024-11-12T12:25:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:25,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414385075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414385075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414385077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414385077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414385080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,084 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T12:25:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414385191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414385191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414385191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414385191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414385401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414385402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414385402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414385402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,411 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/cd4a0a7fcad34a6a979526b0edb44172 2024-11-12T12:25:25,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/2e2c1b475bcc400baa8579fe2eaf1dca is 50, key is test_row_0/B:col10/1731414322910/Put/seqid=0 2024-11-12T12:25:25,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742075_1251 (size=12001) 2024-11-12T12:25:25,423 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/2e2c1b475bcc400baa8579fe2eaf1dca 2024-11-12T12:25:25,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/cd3cfad0fda74312a29c1f0a976eb2c3 is 50, key is test_row_0/C:col10/1731414322910/Put/seqid=0 2024-11-12T12:25:25,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742076_1252 (size=12001) 2024-11-12T12:25:25,435 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/cd3cfad0fda74312a29c1f0a976eb2c3 2024-11-12T12:25:25,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/cd4a0a7fcad34a6a979526b0edb44172 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cd4a0a7fcad34a6a979526b0edb44172 2024-11-12T12:25:25,443 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cd4a0a7fcad34a6a979526b0edb44172, entries=150, sequenceid=76, filesize=11.7 K 2024-11-12T12:25:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/2e2c1b475bcc400baa8579fe2eaf1dca as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2e2c1b475bcc400baa8579fe2eaf1dca 2024-11-12T12:25:25,448 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2e2c1b475bcc400baa8579fe2eaf1dca, entries=150, sequenceid=76, filesize=11.7 K 2024-11-12T12:25:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/cd3cfad0fda74312a29c1f0a976eb2c3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cd3cfad0fda74312a29c1f0a976eb2c3 2024-11-12T12:25:25,452 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cd3cfad0fda74312a29c1f0a976eb2c3, entries=150, sequenceid=76, filesize=11.7 K 2024-11-12T12:25:25,453 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d1a5c014bc05d48b1ac89d4e1685996f in 451ms, sequenceid=76, compaction requested=false 2024-11-12T12:25:25,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:25,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:25,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-12T12:25:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-12T12:25:25,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-12T12:25:25,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 604 msec 2024-11-12T12:25:25,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 625 msec 2024-11-12T12:25:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T12:25:25,471 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:25,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-12T12:25:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T12:25:25,473 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:25,473 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:25,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T12:25:25,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T12:25:25,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:25,626 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-12T12:25:25,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/5f6f5dc0b2aa401498fb08498ab25c69 is 50, key is test_row_0/A:col10/1731414325075/Put/seqid=0 2024-11-12T12:25:25,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742077_1253 (size=12001) 2024-11-12T12:25:25,636 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/5f6f5dc0b2aa401498fb08498ab25c69 2024-11-12T12:25:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/77e3cbf6c6ca4529b32e5c0d1f1f9da7 is 50, key is test_row_0/B:col10/1731414325075/Put/seqid=0 2024-11-12T12:25:25,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742078_1254 (size=12001) 2024-11-12T12:25:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:25,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:25,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414385729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414385731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414385731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414385732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T12:25:25,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414385841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414385841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414385841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414385841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,047 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/77e3cbf6c6ca4529b32e5c0d1f1f9da7 2024-11-12T12:25:26,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/0fcf791592eb4be198b3a58b04848180 is 50, key is test_row_0/C:col10/1731414325075/Put/seqid=0 2024-11-12T12:25:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414386051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414386051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414386051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414386051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742079_1255 (size=12001) 2024-11-12T12:25:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T12:25:26,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414386361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414386361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414386361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414386361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,458 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/0fcf791592eb4be198b3a58b04848180 2024-11-12T12:25:26,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/5f6f5dc0b2aa401498fb08498ab25c69 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5f6f5dc0b2aa401498fb08498ab25c69 2024-11-12T12:25:26,574 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5f6f5dc0b2aa401498fb08498ab25c69, entries=150, sequenceid=91, filesize=11.7 K 2024-11-12T12:25:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/77e3cbf6c6ca4529b32e5c0d1f1f9da7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/77e3cbf6c6ca4529b32e5c0d1f1f9da7 2024-11-12T12:25:26,579 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/77e3cbf6c6ca4529b32e5c0d1f1f9da7, entries=150, sequenceid=91, filesize=11.7 K 2024-11-12T12:25:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/0fcf791592eb4be198b3a58b04848180 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/0fcf791592eb4be198b3a58b04848180 2024-11-12T12:25:26,583 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/0fcf791592eb4be198b3a58b04848180, entries=150, sequenceid=91, filesize=11.7 K 2024-11-12T12:25:26,583 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d1a5c014bc05d48b1ac89d4e1685996f in 957ms, sequenceid=91, compaction requested=true 2024-11-12T12:25:26,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:26,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:26,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-12T12:25:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-12T12:25:26,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-12T12:25:26,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1110 sec 2024-11-12T12:25:26,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1150 sec 2024-11-12T12:25:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T12:25:26,611 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:26,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-12T12:25:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T12:25:26,613 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:26,614 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:26,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T12:25:26,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T12:25:26,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:26,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-12T12:25:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/fa213122b98e4349972edf41d1f24bea is 50, key is test_row_0/A:col10/1731414325731/Put/seqid=0 2024-11-12T12:25:26,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742080_1256 (size=12001) 2024-11-12T12:25:26,781 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/fa213122b98e4349972edf41d1f24bea 2024-11-12T12:25:26,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/9a8c443550d048249a6c010ea54945e4 is 50, key is test_row_0/B:col10/1731414325731/Put/seqid=0 2024-11-12T12:25:26,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742081_1257 (size=12001) 2024-11-12T12:25:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:26,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:26,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414386883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414386886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414386887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414386887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T12:25:26,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414386991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:26,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:26,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414387001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414387002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414387002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414387111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,114 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,193 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/9a8c443550d048249a6c010ea54945e4 2024-11-12T12:25:27,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/269ba313c0344a308238213005f46cae is 50, key is test_row_0/C:col10/1731414325731/Put/seqid=0 2024-11-12T12:25:27,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414387202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742082_1258 (size=12001) 2024-11-12T12:25:27,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414387211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414387211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414387211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T12:25:27,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414387511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414387521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414387521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414387521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:27,605 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/269ba313c0344a308238213005f46cae 2024-11-12T12:25:27,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/fa213122b98e4349972edf41d1f24bea as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/fa213122b98e4349972edf41d1f24bea 2024-11-12T12:25:27,613 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/fa213122b98e4349972edf41d1f24bea, entries=150, sequenceid=112, filesize=11.7 K 2024-11-12T12:25:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/9a8c443550d048249a6c010ea54945e4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9a8c443550d048249a6c010ea54945e4 2024-11-12T12:25:27,618 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9a8c443550d048249a6c010ea54945e4, entries=150, sequenceid=112, filesize=11.7 K 2024-11-12T12:25:27,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/269ba313c0344a308238213005f46cae as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/269ba313c0344a308238213005f46cae 2024-11-12T12:25:27,622 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/269ba313c0344a308238213005f46cae, entries=150, sequenceid=112, filesize=11.7 K 2024-11-12T12:25:27,623 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for d1a5c014bc05d48b1ac89d4e1685996f in 857ms, sequenceid=112, compaction requested=true 2024-11-12T12:25:27,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:27,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:27,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-12T12:25:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-12T12:25:27,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-12T12:25:27,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0110 sec 2024-11-12T12:25:27,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0160 sec 2024-11-12T12:25:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T12:25:27,761 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:27,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-12T12:25:27,762 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T12:25:27,763 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:27,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T12:25:27,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T12:25:27,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:27,915 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-12T12:25:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/85cc9556ef05463c856d7f280f668e25 is 50, key is test_row_0/A:col10/1731414326879/Put/seqid=0 2024-11-12T12:25:27,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742083_1259 (size=12001) 2024-11-12T12:25:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:28,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414388048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414388048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414388049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414388051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T12:25:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414388161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414388161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414388161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414388161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,324 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/85cc9556ef05463c856d7f280f668e25 2024-11-12T12:25:28,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/b9a116a86e474189a39b490d4fb8dab9 is 50, key is test_row_0/B:col10/1731414326879/Put/seqid=0 2024-11-12T12:25:28,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742084_1260 (size=12001) 2024-11-12T12:25:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414388371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414388371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414388371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414388371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T12:25:28,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414388681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414388682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414388682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414388682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:28,738 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/b9a116a86e474189a39b490d4fb8dab9 2024-11-12T12:25:28,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/1d5a51786d3a4b04991518cfc9327deb is 50, key is test_row_0/C:col10/1731414326879/Put/seqid=0 2024-11-12T12:25:28,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742085_1261 (size=12001) 2024-11-12T12:25:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T12:25:29,153 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/1d5a51786d3a4b04991518cfc9327deb 2024-11-12T12:25:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/85cc9556ef05463c856d7f280f668e25 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/85cc9556ef05463c856d7f280f668e25 2024-11-12T12:25:29,161 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/85cc9556ef05463c856d7f280f668e25, entries=150, sequenceid=128, filesize=11.7 K 2024-11-12T12:25:29,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/b9a116a86e474189a39b490d4fb8dab9 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b9a116a86e474189a39b490d4fb8dab9 2024-11-12T12:25:29,165 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b9a116a86e474189a39b490d4fb8dab9, entries=150, sequenceid=128, filesize=11.7 K 2024-11-12T12:25:29,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/1d5a51786d3a4b04991518cfc9327deb as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/1d5a51786d3a4b04991518cfc9327deb 2024-11-12T12:25:29,170 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/1d5a51786d3a4b04991518cfc9327deb, entries=150, sequenceid=128, filesize=11.7 K 2024-11-12T12:25:29,171 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d1a5c014bc05d48b1ac89d4e1685996f in 1256ms, sequenceid=128, compaction requested=true 2024-11-12T12:25:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-12T12:25:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-12T12:25:29,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-12T12:25:29,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-12T12:25:29,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4180 sec 2024-11-12T12:25:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:29,195 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:29,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-12T12:25:29,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/a4fb504fbe91472196c34c65aba6771d is 50, key is test_row_0/A:col10/1731414329195/Put/seqid=0 2024-11-12T12:25:29,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742086_1262 (size=12151) 2024-11-12T12:25:29,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414389209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414389210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414389211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414389216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414389321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414389321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414389321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414389331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414389531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414389531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414389531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414389541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/a4fb504fbe91472196c34c65aba6771d 2024-11-12T12:25:29,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/467407d86ffa44a891fbc9719927c58e is 50, key is test_row_0/B:col10/1731414329195/Put/seqid=0 2024-11-12T12:25:29,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742087_1263 (size=12151) 2024-11-12T12:25:29,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414389841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414389841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414389841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414389851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T12:25:29,911 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:29,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-12T12:25:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T12:25:29,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:29,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:29,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:30,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/467407d86ffa44a891fbc9719927c58e 2024-11-12T12:25:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T12:25:30,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/cdc0637bc89440dcb64c81c48be6cc44 is 50, key is test_row_0/C:col10/1731414329195/Put/seqid=0 2024-11-12T12:25:30,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742088_1264 (size=12151) 2024-11-12T12:25:30,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T12:25:30,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:30,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:30,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T12:25:30,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:30,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:30,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T12:25:30,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414390351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414390351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414390351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:30,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T12:25:30,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:30,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:30,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:30,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414390371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:30,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/cdc0637bc89440dcb64c81c48be6cc44 2024-11-12T12:25:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/a4fb504fbe91472196c34c65aba6771d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a4fb504fbe91472196c34c65aba6771d 2024-11-12T12:25:30,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a4fb504fbe91472196c34c65aba6771d, entries=150, sequenceid=150, filesize=11.9 K 2024-11-12T12:25:30,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/467407d86ffa44a891fbc9719927c58e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/467407d86ffa44a891fbc9719927c58e 2024-11-12T12:25:30,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/467407d86ffa44a891fbc9719927c58e, entries=150, sequenceid=150, filesize=11.9 K 2024-11-12T12:25:30,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/cdc0637bc89440dcb64c81c48be6cc44 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cdc0637bc89440dcb64c81c48be6cc44 2024-11-12T12:25:30,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cdc0637bc89440dcb64c81c48be6cc44, entries=150, sequenceid=150, filesize=11.9 K 2024-11-12T12:25:30,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d1a5c014bc05d48b1ac89d4e1685996f in 1272ms, sequenceid=150, compaction requested=true 2024-11-12T12:25:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:30,468 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-12T12:25:30,468 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-12T12:25:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:30,472 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72259 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-12T12:25:30,472 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/B is initiating minor compaction (all files) 2024-11-12T12:25:30,472 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/B in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,472 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2cbee7930239440299cb137a5af94beb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2e2c1b475bcc400baa8579fe2eaf1dca, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/77e3cbf6c6ca4529b32e5c0d1f1f9da7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9a8c443550d048249a6c010ea54945e4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b9a116a86e474189a39b490d4fb8dab9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/467407d86ffa44a891fbc9719927c58e] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=70.6 K 2024-11-12T12:25:30,473 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72259 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-12T12:25:30,473 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/A is initiating minor compaction (all files) 2024-11-12T12:25:30,474 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/A in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,474 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/c9ef1107952d4142ae600bdcd3c76c3b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cd4a0a7fcad34a6a979526b0edb44172, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5f6f5dc0b2aa401498fb08498ab25c69, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/fa213122b98e4349972edf41d1f24bea, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/85cc9556ef05463c856d7f280f668e25, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a4fb504fbe91472196c34c65aba6771d] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=70.6 K 2024-11-12T12:25:30,476 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9ef1107952d4142ae600bdcd3c76c3b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414322882 2024-11-12T12:25:30,476 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cbee7930239440299cb137a5af94beb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414322882 2024-11-12T12:25:30,477 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd4a0a7fcad34a6a979526b0edb44172, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731414322910 2024-11-12T12:25:30,478 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e2c1b475bcc400baa8579fe2eaf1dca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731414322910 2024-11-12T12:25:30,478 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6f5dc0b2aa401498fb08498ab25c69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731414325070 2024-11-12T12:25:30,479 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 77e3cbf6c6ca4529b32e5c0d1f1f9da7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731414325070 2024-11-12T12:25:30,479 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa213122b98e4349972edf41d1f24bea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731414325724 2024-11-12T12:25:30,479 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a8c443550d048249a6c010ea54945e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731414325724 2024-11-12T12:25:30,480 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85cc9556ef05463c856d7f280f668e25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731414326879 2024-11-12T12:25:30,480 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a116a86e474189a39b490d4fb8dab9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731414326879 2024-11-12T12:25:30,480 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4fb504fbe91472196c34c65aba6771d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731414328044 2024-11-12T12:25:30,481 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 467407d86ffa44a891fbc9719927c58e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731414328044 2024-11-12T12:25:30,514 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#A#compaction#219 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:30,514 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/92701e36a4c34b559fbf23cdd6b39273 is 50, key is test_row_0/A:col10/1731414329195/Put/seqid=0 2024-11-12T12:25:30,523 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#B#compaction#220 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:30,524 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4541547028c0409e9ae7129220dbe0f8 is 50, key is test_row_0/B:col10/1731414329195/Put/seqid=0 2024-11-12T12:25:30,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T12:25:30,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,529 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-12T12:25:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T12:25:30,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/5b03ba80fad54c2e99b73ba18f6f62ae is 50, key is test_row_0/A:col10/1731414329215/Put/seqid=0 2024-11-12T12:25:30,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742090_1266 (size=12459) 2024-11-12T12:25:30,573 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4541547028c0409e9ae7129220dbe0f8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4541547028c0409e9ae7129220dbe0f8 2024-11-12T12:25:30,581 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/B of d1a5c014bc05d48b1ac89d4e1685996f into 4541547028c0409e9ae7129220dbe0f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:30,581 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:30,581 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/B, priority=10, startTime=1731414330468; duration=0sec 2024-11-12T12:25:30,581 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:30,581 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:B 2024-11-12T12:25:30,581 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-12T12:25:30,584 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72259 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-12T12:25:30,584 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/C is initiating minor compaction (all files) 2024-11-12T12:25:30,584 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/C in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:30,584 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/88b9520ff40840038ef3197be90bed8f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cd3cfad0fda74312a29c1f0a976eb2c3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/0fcf791592eb4be198b3a58b04848180, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/269ba313c0344a308238213005f46cae, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/1d5a51786d3a4b04991518cfc9327deb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cdc0637bc89440dcb64c81c48be6cc44] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=70.6 K 2024-11-12T12:25:30,585 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b9520ff40840038ef3197be90bed8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414322882 2024-11-12T12:25:30,586 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting cd3cfad0fda74312a29c1f0a976eb2c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731414322910 2024-11-12T12:25:30,586 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fcf791592eb4be198b3a58b04848180, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731414325070 2024-11-12T12:25:30,586 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 269ba313c0344a308238213005f46cae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731414325724 2024-11-12T12:25:30,587 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d5a51786d3a4b04991518cfc9327deb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731414326879 2024-11-12T12:25:30,587 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting cdc0637bc89440dcb64c81c48be6cc44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731414328044 2024-11-12T12:25:30,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742089_1265 (size=12459) 2024-11-12T12:25:30,596 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/92701e36a4c34b559fbf23cdd6b39273 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/92701e36a4c34b559fbf23cdd6b39273 2024-11-12T12:25:30,602 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/A of d1a5c014bc05d48b1ac89d4e1685996f into 92701e36a4c34b559fbf23cdd6b39273(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:30,603 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:30,603 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/A, priority=10, startTime=1731414330468; duration=0sec 2024-11-12T12:25:30,603 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:30,603 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:A 2024-11-12T12:25:30,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742091_1267 (size=12151) 2024-11-12T12:25:30,620 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#C#compaction#222 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:30,620 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/4aebf4797e5e492c9f9799ba034f555c is 50, key is test_row_0/C:col10/1731414329195/Put/seqid=0 2024-11-12T12:25:30,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742092_1268 (size=12459) 2024-11-12T12:25:31,022 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/5b03ba80fad54c2e99b73ba18f6f62ae 2024-11-12T12:25:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/dc50ad5375c94206bc535fe4c76682f8 is 50, key is test_row_0/B:col10/1731414329215/Put/seqid=0 2024-11-12T12:25:31,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742093_1269 (size=12151) 2024-11-12T12:25:31,037 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/dc50ad5375c94206bc535fe4c76682f8 2024-11-12T12:25:31,042 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/4aebf4797e5e492c9f9799ba034f555c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4aebf4797e5e492c9f9799ba034f555c 2024-11-12T12:25:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/db0cd70b03ed43ba94a2db96a3b7646d is 50, key is test_row_0/C:col10/1731414329215/Put/seqid=0 2024-11-12T12:25:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T12:25:31,052 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/C of d1a5c014bc05d48b1ac89d4e1685996f into 4aebf4797e5e492c9f9799ba034f555c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:31,052 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:31,052 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/C, priority=10, startTime=1731414330468; duration=0sec 2024-11-12T12:25:31,052 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:31,052 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:C 2024-11-12T12:25:31,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742094_1270 (size=12151) 2024-11-12T12:25:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414391228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:31,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414391341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:31,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414391361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:31,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414391361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:31,365 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:31,365 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:31,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414391371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:31,378 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414391391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:31,394 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:31,468 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/db0cd70b03ed43ba94a2db96a3b7646d 2024-11-12T12:25:31,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/5b03ba80fad54c2e99b73ba18f6f62ae as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5b03ba80fad54c2e99b73ba18f6f62ae 2024-11-12T12:25:31,476 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5b03ba80fad54c2e99b73ba18f6f62ae, entries=150, sequenceid=164, filesize=11.9 K 2024-11-12T12:25:31,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/dc50ad5375c94206bc535fe4c76682f8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/dc50ad5375c94206bc535fe4c76682f8 2024-11-12T12:25:31,482 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/dc50ad5375c94206bc535fe4c76682f8, entries=150, sequenceid=164, filesize=11.9 K 2024-11-12T12:25:31,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/db0cd70b03ed43ba94a2db96a3b7646d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/db0cd70b03ed43ba94a2db96a3b7646d 2024-11-12T12:25:31,489 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/db0cd70b03ed43ba94a2db96a3b7646d, entries=150, sequenceid=164, filesize=11.9 K 2024-11-12T12:25:31,490 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d1a5c014bc05d48b1ac89d4e1685996f in 962ms, sequenceid=164, compaction requested=false 2024-11-12T12:25:31,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:31,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:31,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-12T12:25:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-12T12:25:31,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-12T12:25:31,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5790 sec 2024-11-12T12:25:31,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5850 sec 2024-11-12T12:25:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:31,553 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:31,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-12T12:25:31,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/40ac58fb634148e1876284caaf368bf0 is 50, key is test_row_0/A:col10/1731414331225/Put/seqid=0 2024-11-12T12:25:31,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742095_1271 (size=14541) 2024-11-12T12:25:31,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/40ac58fb634148e1876284caaf368bf0 2024-11-12T12:25:31,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/f27d39300f8b4559bf77b535a7dd0b1a is 50, key is test_row_0/B:col10/1731414331225/Put/seqid=0 2024-11-12T12:25:31,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414391602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:31,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742096_1272 (size=12151) 2024-11-12T12:25:31,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414391712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:31,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414391921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:32,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/f27d39300f8b4559bf77b535a7dd0b1a 2024-11-12T12:25:32,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/d37098c5e1744db7a715a0c08ec61cbc is 50, key is test_row_0/C:col10/1731414331225/Put/seqid=0 2024-11-12T12:25:32,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742097_1273 (size=12151) 2024-11-12T12:25:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T12:25:32,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/d37098c5e1744db7a715a0c08ec61cbc 2024-11-12T12:25:32,061 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:32,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-12T12:25:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T12:25:32,065 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:32,065 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:32,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:32,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/40ac58fb634148e1876284caaf368bf0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/40ac58fb634148e1876284caaf368bf0 2024-11-12T12:25:32,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/40ac58fb634148e1876284caaf368bf0, entries=200, sequenceid=190, filesize=14.2 K 2024-11-12T12:25:32,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/f27d39300f8b4559bf77b535a7dd0b1a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f27d39300f8b4559bf77b535a7dd0b1a 2024-11-12T12:25:32,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f27d39300f8b4559bf77b535a7dd0b1a, entries=150, sequenceid=190, filesize=11.9 K 2024-11-12T12:25:32,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/d37098c5e1744db7a715a0c08ec61cbc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d37098c5e1744db7a715a0c08ec61cbc 2024-11-12T12:25:32,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d37098c5e1744db7a715a0c08ec61cbc, entries=150, sequenceid=190, filesize=11.9 K 2024-11-12T12:25:32,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d1a5c014bc05d48b1ac89d4e1685996f in 529ms, sequenceid=190, compaction requested=true 2024-11-12T12:25:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:32,082 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:32,082 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:32,084 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:32,084 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/B is initiating minor compaction (all files) 2024-11-12T12:25:32,084 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/B in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:32,084 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4541547028c0409e9ae7129220dbe0f8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/dc50ad5375c94206bc535fe4c76682f8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f27d39300f8b4559bf77b535a7dd0b1a] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=35.9 K 2024-11-12T12:25:32,084 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:32,084 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/A is initiating minor compaction (all files) 2024-11-12T12:25:32,084 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/A in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:32,084 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/92701e36a4c34b559fbf23cdd6b39273, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5b03ba80fad54c2e99b73ba18f6f62ae, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/40ac58fb634148e1876284caaf368bf0] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=38.2 K 2024-11-12T12:25:32,085 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92701e36a4c34b559fbf23cdd6b39273, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731414328044 2024-11-12T12:25:32,086 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 4541547028c0409e9ae7129220dbe0f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731414328044 2024-11-12T12:25:32,086 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b03ba80fad54c2e99b73ba18f6f62ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731414329208 2024-11-12T12:25:32,086 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting dc50ad5375c94206bc535fe4c76682f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731414329208 2024-11-12T12:25:32,086 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40ac58fb634148e1876284caaf368bf0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731414331192 2024-11-12T12:25:32,086 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting f27d39300f8b4559bf77b535a7dd0b1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731414331192 2024-11-12T12:25:32,098 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:32,098 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/3c54b1fd57014069b8b89ce530d79f30 is 50, key is test_row_0/B:col10/1731414331225/Put/seqid=0 2024-11-12T12:25:32,100 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#A#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:32,101 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/cc17dd929fba4d5da7cc205ab43b2531 is 50, key is test_row_0/A:col10/1731414331225/Put/seqid=0 2024-11-12T12:25:32,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742099_1275 (size=12561) 2024-11-12T12:25:32,115 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/cc17dd929fba4d5da7cc205ab43b2531 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cc17dd929fba4d5da7cc205ab43b2531 2024-11-12T12:25:32,121 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/A of d1a5c014bc05d48b1ac89d4e1685996f into cc17dd929fba4d5da7cc205ab43b2531(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:32,121 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:32,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742098_1274 (size=12561) 2024-11-12T12:25:32,121 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/A, priority=13, startTime=1731414332082; duration=0sec 2024-11-12T12:25:32,121 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:32,121 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:A 2024-11-12T12:25:32,121 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:32,123 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:32,123 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/C is initiating minor compaction (all files) 2024-11-12T12:25:32,123 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/C in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:32,123 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4aebf4797e5e492c9f9799ba034f555c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/db0cd70b03ed43ba94a2db96a3b7646d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d37098c5e1744db7a715a0c08ec61cbc] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=35.9 K 2024-11-12T12:25:32,124 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aebf4797e5e492c9f9799ba034f555c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731414328044 2024-11-12T12:25:32,124 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting db0cd70b03ed43ba94a2db96a3b7646d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731414329208 2024-11-12T12:25:32,124 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting d37098c5e1744db7a715a0c08ec61cbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731414331192 2024-11-12T12:25:32,162 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:32,163 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/fc6e0eef95064e57b96f7eff763e503d is 50, key is test_row_0/C:col10/1731414331225/Put/seqid=0 2024-11-12T12:25:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T12:25:32,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T12:25:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:32,219 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-12T12:25:32,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742100_1276 (size=12561) 2024-11-12T12:25:32,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:32,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/0671abb8642c434ca92483d9f91fcf17 is 50, key is test_row_0/A:col10/1731414331589/Put/seqid=0 2024-11-12T12:25:32,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742101_1277 (size=12151) 2024-11-12T12:25:32,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414392354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T12:25:32,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414392462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:32,528 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/3c54b1fd57014069b8b89ce530d79f30 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/3c54b1fd57014069b8b89ce530d79f30 2024-11-12T12:25:32,534 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/B of d1a5c014bc05d48b1ac89d4e1685996f into 3c54b1fd57014069b8b89ce530d79f30(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:32,534 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:32,534 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/B, priority=13, startTime=1731414332082; duration=0sec 2024-11-12T12:25:32,535 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:32,535 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:B 2024-11-12T12:25:32,630 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/fc6e0eef95064e57b96f7eff763e503d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/fc6e0eef95064e57b96f7eff763e503d 2024-11-12T12:25:32,638 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/C of d1a5c014bc05d48b1ac89d4e1685996f into fc6e0eef95064e57b96f7eff763e503d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:32,638 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:32,638 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/C, priority=13, startTime=1731414332082; duration=0sec 2024-11-12T12:25:32,638 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:32,638 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:C 2024-11-12T12:25:32,671 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/0671abb8642c434ca92483d9f91fcf17 2024-11-12T12:25:32,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414392671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:32,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4089539f4918497ea09aeca19e4810ff is 50, key is test_row_0/B:col10/1731414331589/Put/seqid=0 2024-11-12T12:25:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T12:25:32,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742102_1278 (size=12151) 2024-11-12T12:25:32,702 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4089539f4918497ea09aeca19e4810ff 2024-11-12T12:25:32,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/225965fde5c74d41b306773924b3e9d5 is 50, key is test_row_0/C:col10/1731414331589/Put/seqid=0 2024-11-12T12:25:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742103_1279 (size=12151) 2024-11-12T12:25:32,742 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/225965fde5c74d41b306773924b3e9d5 2024-11-12T12:25:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/0671abb8642c434ca92483d9f91fcf17 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0671abb8642c434ca92483d9f91fcf17 2024-11-12T12:25:32,753 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0671abb8642c434ca92483d9f91fcf17, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T12:25:32,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4089539f4918497ea09aeca19e4810ff as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4089539f4918497ea09aeca19e4810ff 2024-11-12T12:25:32,760 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4089539f4918497ea09aeca19e4810ff, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T12:25:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/225965fde5c74d41b306773924b3e9d5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/225965fde5c74d41b306773924b3e9d5 2024-11-12T12:25:32,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/225965fde5c74d41b306773924b3e9d5, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T12:25:32,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d1a5c014bc05d48b1ac89d4e1685996f in 547ms, sequenceid=204, compaction requested=false 2024-11-12T12:25:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-12T12:25:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-12T12:25:32,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-12T12:25:32,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 703 msec 2024-11-12T12:25:32,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 709 msec 2024-11-12T12:25:32,985 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:32,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-12T12:25:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:33,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/19624aa0070a453cafdfdc48eb098cb4 is 50, key is test_row_0/A:col10/1731414332343/Put/seqid=0 2024-11-12T12:25:33,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742104_1280 (size=14541) 2024-11-12T12:25:33,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414393042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:33,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414393151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T12:25:33,201 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:33,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-12T12:25:33,203 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T12:25:33,204 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:33,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T12:25:33,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T12:25:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414393361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:33,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414393381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414393381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:33,384 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:33,384 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:33,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:33,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414393401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:33,403 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:33,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414393411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:33,413 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in d1a5c014bc05d48b1ac89d4e1685996f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:33,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/19624aa0070a453cafdfdc48eb098cb4 2024-11-12T12:25:33,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/ac58b1534961427488aa741e00924748 is 50, key is test_row_0/B:col10/1731414332343/Put/seqid=0 2024-11-12T12:25:33,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742105_1281 (size=12151) 2024-11-12T12:25:33,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T12:25:33,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:33,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T12:25:33,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T12:25:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414393671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:33,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T12:25:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T12:25:33,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/ac58b1534961427488aa741e00924748 2024-11-12T12:25:33,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/dcbd85b318e649cc972366abc1f64de9 is 50, key is test_row_0/C:col10/1731414332343/Put/seqid=0 2024-11-12T12:25:33,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742106_1282 (size=12151) 2024-11-12T12:25:33,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T12:25:33,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:33,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:33,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:34,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T12:25:34,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:34,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:34,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:34,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:34,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:34,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:34,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414394181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:34,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/dcbd85b318e649cc972366abc1f64de9 2024-11-12T12:25:34,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/19624aa0070a453cafdfdc48eb098cb4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/19624aa0070a453cafdfdc48eb098cb4 2024-11-12T12:25:34,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T12:25:34,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:34,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:34,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:34,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:34,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:34,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/19624aa0070a453cafdfdc48eb098cb4, entries=200, sequenceid=230, filesize=14.2 K 2024-11-12T12:25:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:34,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/ac58b1534961427488aa741e00924748 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/ac58b1534961427488aa741e00924748 2024-11-12T12:25:34,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/ac58b1534961427488aa741e00924748, entries=150, sequenceid=230, filesize=11.9 K 2024-11-12T12:25:34,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/dcbd85b318e649cc972366abc1f64de9 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/dcbd85b318e649cc972366abc1f64de9 2024-11-12T12:25:34,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/dcbd85b318e649cc972366abc1f64de9, entries=150, sequenceid=230, filesize=11.9 K 2024-11-12T12:25:34,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d1a5c014bc05d48b1ac89d4e1685996f in 1307ms, sequenceid=230, compaction requested=true 2024-11-12T12:25:34,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:34,292 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:34,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:34,293 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:34,293 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/A is initiating minor compaction (all files) 2024-11-12T12:25:34,294 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/A in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:34,294 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cc17dd929fba4d5da7cc205ab43b2531, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0671abb8642c434ca92483d9f91fcf17, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/19624aa0070a453cafdfdc48eb098cb4] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=38.3 K 2024-11-12T12:25:34,294 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc17dd929fba4d5da7cc205ab43b2531, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731414331192 2024-11-12T12:25:34,295 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0671abb8642c434ca92483d9f91fcf17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731414331583 2024-11-12T12:25:34,295 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19624aa0070a453cafdfdc48eb098cb4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731414332317 2024-11-12T12:25:34,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:34,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:34,299 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:34,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:34,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:34,300 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:34,301 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/B is initiating minor compaction (all files) 2024-11-12T12:25:34,301 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/B in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:34,301 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/3c54b1fd57014069b8b89ce530d79f30, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4089539f4918497ea09aeca19e4810ff, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/ac58b1534961427488aa741e00924748] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.0 K 2024-11-12T12:25:34,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:34,301 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c54b1fd57014069b8b89ce530d79f30, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731414331192 2024-11-12T12:25:34,305 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:34,305 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/431ca5e307634f61a5948ed0d68aa4ed is 50, key is test_row_0/A:col10/1731414332343/Put/seqid=0 2024-11-12T12:25:34,306 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 4089539f4918497ea09aeca19e4810ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731414331583 2024-11-12T12:25:34,307 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ac58b1534961427488aa741e00924748, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731414332343 2024-11-12T12:25:34,317 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:34,318 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/5b589af3833d4fdb93c0e90facfb3cf3 is 50, key is test_row_0/B:col10/1731414332343/Put/seqid=0 2024-11-12T12:25:34,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742107_1283 (size=12663) 2024-11-12T12:25:34,336 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/431ca5e307634f61a5948ed0d68aa4ed as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/431ca5e307634f61a5948ed0d68aa4ed 2024-11-12T12:25:34,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742108_1284 (size=12663) 2024-11-12T12:25:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T12:25:34,348 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/5b589af3833d4fdb93c0e90facfb3cf3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/5b589af3833d4fdb93c0e90facfb3cf3 2024-11-12T12:25:34,349 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/A of d1a5c014bc05d48b1ac89d4e1685996f into 431ca5e307634f61a5948ed0d68aa4ed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:34,349 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:34,349 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/A, priority=13, startTime=1731414334292; duration=0sec 2024-11-12T12:25:34,349 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:34,349 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:A 2024-11-12T12:25:34,349 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:34,352 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:34,352 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/C is initiating minor compaction (all files) 2024-11-12T12:25:34,352 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/C in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:34,352 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/fc6e0eef95064e57b96f7eff763e503d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/225965fde5c74d41b306773924b3e9d5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/dcbd85b318e649cc972366abc1f64de9] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.0 K 2024-11-12T12:25:34,353 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc6e0eef95064e57b96f7eff763e503d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731414331192 2024-11-12T12:25:34,353 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 225965fde5c74d41b306773924b3e9d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731414331583 2024-11-12T12:25:34,355 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcbd85b318e649cc972366abc1f64de9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731414332343 2024-11-12T12:25:34,357 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/B of d1a5c014bc05d48b1ac89d4e1685996f into 5b589af3833d4fdb93c0e90facfb3cf3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:34,357 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:34,357 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/B, priority=13, startTime=1731414334299; duration=0sec 2024-11-12T12:25:34,357 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:34,357 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:B 2024-11-12T12:25:34,369 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:34,370 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/bebba284494f435ea7ae163a0938f3a5 is 50, key is test_row_0/C:col10/1731414332343/Put/seqid=0 2024-11-12T12:25:34,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742109_1285 (size=12663) 2024-11-12T12:25:34,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T12:25:34,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:34,430 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-12T12:25:34,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/7a0cf6f6a9954a33a324b839058bc803 is 50, key is test_row_0/A:col10/1731414333030/Put/seqid=0 2024-11-12T12:25:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742110_1286 (size=12151) 2024-11-12T12:25:34,799 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/bebba284494f435ea7ae163a0938f3a5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bebba284494f435ea7ae163a0938f3a5 2024-11-12T12:25:34,804 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/C of d1a5c014bc05d48b1ac89d4e1685996f into bebba284494f435ea7ae163a0938f3a5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:34,804 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:34,804 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/C, priority=13, startTime=1731414334300; duration=0sec 2024-11-12T12:25:34,804 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:34,804 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:C 2024-11-12T12:25:34,840 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/7a0cf6f6a9954a33a324b839058bc803 2024-11-12T12:25:34,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/293d27442afd4b46a44d4eb224bc25dc is 50, key is test_row_0/B:col10/1731414333030/Put/seqid=0 2024-11-12T12:25:34,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742111_1287 (size=12151) 2024-11-12T12:25:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:35,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:35,252 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/293d27442afd4b46a44d4eb224bc25dc 2024-11-12T12:25:35,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/7e1629d55adc436f88a130b328f2b071 is 50, key is test_row_0/C:col10/1731414333030/Put/seqid=0 2024-11-12T12:25:35,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742112_1288 (size=12151) 2024-11-12T12:25:35,273 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/7e1629d55adc436f88a130b328f2b071 2024-11-12T12:25:35,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/7a0cf6f6a9954a33a324b839058bc803 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7a0cf6f6a9954a33a324b839058bc803 2024-11-12T12:25:35,281 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7a0cf6f6a9954a33a324b839058bc803, entries=150, sequenceid=244, filesize=11.9 K 2024-11-12T12:25:35,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/293d27442afd4b46a44d4eb224bc25dc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/293d27442afd4b46a44d4eb224bc25dc 2024-11-12T12:25:35,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414395279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:35,286 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/293d27442afd4b46a44d4eb224bc25dc, entries=150, sequenceid=244, filesize=11.9 K 2024-11-12T12:25:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/7e1629d55adc436f88a130b328f2b071 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7e1629d55adc436f88a130b328f2b071 2024-11-12T12:25:35,292 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7e1629d55adc436f88a130b328f2b071, entries=150, sequenceid=244, filesize=11.9 K 2024-11-12T12:25:35,292 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d1a5c014bc05d48b1ac89d4e1685996f in 862ms, sequenceid=244, compaction requested=false 2024-11-12T12:25:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-12T12:25:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-12T12:25:35,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-12T12:25:35,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0900 sec 2024-11-12T12:25:35,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.0950 sec 2024-11-12T12:25:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T12:25:35,352 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:35,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-12T12:25:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T12:25:35,354 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:35,355 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:35,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:35,394 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:35,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:25:35,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/71873c3153d047cc928ef5d0aea0edab is 50, key is test_row_0/A:col10/1731414335393/Put/seqid=0 2024-11-12T12:25:35,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742113_1289 (size=14741) 2024-11-12T12:25:35,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/71873c3153d047cc928ef5d0aea0edab 2024-11-12T12:25:35,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/8738d56428404851a24677c0a0ac59c1 is 50, key is test_row_0/B:col10/1731414335393/Put/seqid=0 2024-11-12T12:25:35,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742114_1290 (size=12301) 2024-11-12T12:25:35,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/8738d56428404851a24677c0a0ac59c1 2024-11-12T12:25:35,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/afe239592c3f43db9381d0c4e73330a4 is 50, key is test_row_0/C:col10/1731414335393/Put/seqid=0 2024-11-12T12:25:35,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742115_1291 (size=12301) 2024-11-12T12:25:35,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/afe239592c3f43db9381d0c4e73330a4 2024-11-12T12:25:35,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414395433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:35,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/71873c3153d047cc928ef5d0aea0edab as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/71873c3153d047cc928ef5d0aea0edab 2024-11-12T12:25:35,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/71873c3153d047cc928ef5d0aea0edab, entries=200, sequenceid=270, filesize=14.4 K 2024-11-12T12:25:35,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/8738d56428404851a24677c0a0ac59c1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/8738d56428404851a24677c0a0ac59c1 2024-11-12T12:25:35,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/8738d56428404851a24677c0a0ac59c1, entries=150, sequenceid=270, filesize=12.0 K 2024-11-12T12:25:35,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/afe239592c3f43db9381d0c4e73330a4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/afe239592c3f43db9381d0c4e73330a4 2024-11-12T12:25:35,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/afe239592c3f43db9381d0c4e73330a4, entries=150, sequenceid=270, filesize=12.0 K 2024-11-12T12:25:35,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d1a5c014bc05d48b1ac89d4e1685996f in 58ms, sequenceid=270, compaction requested=true 2024-11-12T12:25:35,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:35,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:35,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:35,452 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:35,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:35,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:35,452 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:35,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:35,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:35,453 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:35,453 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/B is initiating minor compaction (all files) 2024-11-12T12:25:35,453 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/B in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:35,453 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/5b589af3833d4fdb93c0e90facfb3cf3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/293d27442afd4b46a44d4eb224bc25dc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/8738d56428404851a24677c0a0ac59c1] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.2 K 2024-11-12T12:25:35,454 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:35,454 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/A is initiating minor compaction (all files) 2024-11-12T12:25:35,454 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/A in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:35,455 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/431ca5e307634f61a5948ed0d68aa4ed, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7a0cf6f6a9954a33a324b839058bc803, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/71873c3153d047cc928ef5d0aea0edab] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=38.6 K 2024-11-12T12:25:35,455 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b589af3833d4fdb93c0e90facfb3cf3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731414332343 2024-11-12T12:25:35,455 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 431ca5e307634f61a5948ed0d68aa4ed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731414332343 2024-11-12T12:25:35,455 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 293d27442afd4b46a44d4eb224bc25dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731414333008 2024-11-12T12:25:35,455 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a0cf6f6a9954a33a324b839058bc803, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731414333008 2024-11-12T12:25:35,455 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 8738d56428404851a24677c0a0ac59c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731414335235 2024-11-12T12:25:35,455 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71873c3153d047cc928ef5d0aea0edab, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731414335235 2024-11-12T12:25:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T12:25:35,462 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:35,462 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/f860230330cf423b822c5ed2e803e602 is 50, key is test_row_0/A:col10/1731414335393/Put/seqid=0 2024-11-12T12:25:35,466 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:35,467 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/b941e8402cda488ba11465cab9265d3f is 50, key is test_row_0/B:col10/1731414335393/Put/seqid=0 2024-11-12T12:25:35,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742116_1292 (size=12915) 2024-11-12T12:25:35,479 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/f860230330cf423b822c5ed2e803e602 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/f860230330cf423b822c5ed2e803e602 2024-11-12T12:25:35,483 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/A of d1a5c014bc05d48b1ac89d4e1685996f into f860230330cf423b822c5ed2e803e602(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:35,483 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:35,484 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/A, priority=13, startTime=1731414335452; duration=0sec 2024-11-12T12:25:35,484 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:35,484 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:A 2024-11-12T12:25:35,484 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:35,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742117_1293 (size=12915) 2024-11-12T12:25:35,485 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:35,485 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/C is initiating minor compaction (all files) 2024-11-12T12:25:35,485 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/C in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:35,485 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bebba284494f435ea7ae163a0938f3a5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7e1629d55adc436f88a130b328f2b071, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/afe239592c3f43db9381d0c4e73330a4] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.2 K 2024-11-12T12:25:35,486 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting bebba284494f435ea7ae163a0938f3a5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731414332343 2024-11-12T12:25:35,486 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e1629d55adc436f88a130b328f2b071, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731414333008 2024-11-12T12:25:35,487 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting afe239592c3f43db9381d0c4e73330a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731414335235 2024-11-12T12:25:35,494 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:35,494 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/71b154f396284ec0b2c77d640187af68 is 50, key is test_row_0/C:col10/1731414335393/Put/seqid=0 2024-11-12T12:25:35,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742118_1294 (size=12915) 2024-11-12T12:25:35,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T12:25:35,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:35,507 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:25:35,509 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/71b154f396284ec0b2c77d640187af68 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/71b154f396284ec0b2c77d640187af68 2024-11-12T12:25:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/5766b4ef55d9450fb820dd0f2f67ddfa is 50, key is test_row_0/A:col10/1731414335404/Put/seqid=0 2024-11-12T12:25:35,515 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/C of d1a5c014bc05d48b1ac89d4e1685996f into 71b154f396284ec0b2c77d640187af68(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:35,515 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:35,515 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/C, priority=13, startTime=1731414335452; duration=0sec 2024-11-12T12:25:35,516 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:35,516 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:C 2024-11-12T12:25:35,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742119_1295 (size=12301) 2024-11-12T12:25:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:35,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:35,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414395660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T12:25:35,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414395771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:35,889 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/b941e8402cda488ba11465cab9265d3f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b941e8402cda488ba11465cab9265d3f 2024-11-12T12:25:35,899 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/B of d1a5c014bc05d48b1ac89d4e1685996f into b941e8402cda488ba11465cab9265d3f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:35,899 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:35,899 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/B, priority=13, startTime=1731414335452; duration=0sec 2024-11-12T12:25:35,899 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:35,899 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:B 2024-11-12T12:25:35,922 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/5766b4ef55d9450fb820dd0f2f67ddfa 2024-11-12T12:25:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/bba2b0fa42234d9f883b9a288e8d2780 is 50, key is test_row_0/B:col10/1731414335404/Put/seqid=0 2024-11-12T12:25:35,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742120_1296 (size=12301) 2024-11-12T12:25:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T12:25:35,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414395981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:35,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:36,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414396291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:36,333 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/bba2b0fa42234d9f883b9a288e8d2780 2024-11-12T12:25:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/9bd73b9851774bb485d6f411dd775ae6 is 50, key is test_row_0/C:col10/1731414335404/Put/seqid=0 2024-11-12T12:25:36,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742121_1297 (size=12301) 2024-11-12T12:25:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T12:25:36,751 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/9bd73b9851774bb485d6f411dd775ae6 2024-11-12T12:25:36,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/5766b4ef55d9450fb820dd0f2f67ddfa as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5766b4ef55d9450fb820dd0f2f67ddfa 2024-11-12T12:25:36,759 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5766b4ef55d9450fb820dd0f2f67ddfa, entries=150, sequenceid=282, filesize=12.0 K 2024-11-12T12:25:36,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/bba2b0fa42234d9f883b9a288e8d2780 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/bba2b0fa42234d9f883b9a288e8d2780 2024-11-12T12:25:36,763 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/bba2b0fa42234d9f883b9a288e8d2780, entries=150, sequenceid=282, filesize=12.0 K 2024-11-12T12:25:36,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/9bd73b9851774bb485d6f411dd775ae6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/9bd73b9851774bb485d6f411dd775ae6 2024-11-12T12:25:36,768 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/9bd73b9851774bb485d6f411dd775ae6, entries=150, sequenceid=282, filesize=12.0 K 2024-11-12T12:25:36,769 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d1a5c014bc05d48b1ac89d4e1685996f in 1261ms, sequenceid=282, compaction requested=false 2024-11-12T12:25:36,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:36,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:36,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-12T12:25:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-12T12:25:36,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-12T12:25:36,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-11-12T12:25:36,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4210 sec 2024-11-12T12:25:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:36,803 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:36,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-12T12:25:36,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/d1809ce46a6e47fa9cf25929ba98496d is 50, key is test_row_0/A:col10/1731414336802/Put/seqid=0 2024-11-12T12:25:36,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742122_1298 (size=12301) 2024-11-12T12:25:36,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414396830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414396941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414397151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/d1809ce46a6e47fa9cf25929ba98496d 2024-11-12T12:25:37,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4632180e0a7d44839bf9c1304f6dfd57 is 50, key is test_row_0/B:col10/1731414336802/Put/seqid=0 2024-11-12T12:25:37,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742123_1299 (size=12301) 2024-11-12T12:25:37,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4632180e0a7d44839bf9c1304f6dfd57 2024-11-12T12:25:37,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/d1d30acb41214e80a85dcd536e0efccc is 50, key is test_row_0/C:col10/1731414336802/Put/seqid=0 2024-11-12T12:25:37,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742124_1300 (size=12301) 2024-11-12T12:25:37,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/d1d30acb41214e80a85dcd536e0efccc 2024-11-12T12:25:37,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/d1809ce46a6e47fa9cf25929ba98496d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1809ce46a6e47fa9cf25929ba98496d 2024-11-12T12:25:37,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1809ce46a6e47fa9cf25929ba98496d, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T12:25:37,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/4632180e0a7d44839bf9c1304f6dfd57 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4632180e0a7d44839bf9c1304f6dfd57 2024-11-12T12:25:37,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4632180e0a7d44839bf9c1304f6dfd57, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T12:25:37,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/d1d30acb41214e80a85dcd536e0efccc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d1d30acb41214e80a85dcd536e0efccc 2024-11-12T12:25:37,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d1d30acb41214e80a85dcd536e0efccc, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T12:25:37,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d1a5c014bc05d48b1ac89d4e1685996f in 469ms, sequenceid=310, compaction requested=true 2024-11-12T12:25:37,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:37,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:37,273 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:37,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:37,273 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:37,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:37,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:37,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:37,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:37,277 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:37,277 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:37,278 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/A is initiating minor compaction (all files) 2024-11-12T12:25:37,278 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/B is initiating minor compaction (all files) 2024-11-12T12:25:37,278 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/B in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:37,278 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/A in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:37,278 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b941e8402cda488ba11465cab9265d3f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/bba2b0fa42234d9f883b9a288e8d2780, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4632180e0a7d44839bf9c1304f6dfd57] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.6 K 2024-11-12T12:25:37,278 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/f860230330cf423b822c5ed2e803e602, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5766b4ef55d9450fb820dd0f2f67ddfa, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1809ce46a6e47fa9cf25929ba98496d] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.6 K 2024-11-12T12:25:37,278 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting b941e8402cda488ba11465cab9265d3f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731414335235 2024-11-12T12:25:37,278 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting f860230330cf423b822c5ed2e803e602, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731414335235 2024-11-12T12:25:37,279 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5766b4ef55d9450fb820dd0f2f67ddfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731414335404 2024-11-12T12:25:37,279 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting bba2b0fa42234d9f883b9a288e8d2780, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731414335404 2024-11-12T12:25:37,281 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1809ce46a6e47fa9cf25929ba98496d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731414335654 2024-11-12T12:25:37,281 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 4632180e0a7d44839bf9c1304f6dfd57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731414335654 2024-11-12T12:25:37,294 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#B#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:37,294 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#A#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:37,295 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/0b38d74d94664f35a8791bf0cc0ed39b is 50, key is test_row_0/B:col10/1731414336802/Put/seqid=0 2024-11-12T12:25:37,295 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/1c6037806a984bca91a281bde0e76041 is 50, key is test_row_0/A:col10/1731414336802/Put/seqid=0 2024-11-12T12:25:37,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742125_1301 (size=13017) 2024-11-12T12:25:37,321 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/1c6037806a984bca91a281bde0e76041 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/1c6037806a984bca91a281bde0e76041 2024-11-12T12:25:37,332 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/A of d1a5c014bc05d48b1ac89d4e1685996f into 1c6037806a984bca91a281bde0e76041(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:37,332 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:37,332 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/A, priority=13, startTime=1731414337273; duration=0sec 2024-11-12T12:25:37,332 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:37,332 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:A 2024-11-12T12:25:37,332 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:37,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742126_1302 (size=13017) 2024-11-12T12:25:37,336 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:37,336 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/C is initiating minor compaction (all files) 2024-11-12T12:25:37,336 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/C in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:37,336 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/71b154f396284ec0b2c77d640187af68, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/9bd73b9851774bb485d6f411dd775ae6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d1d30acb41214e80a85dcd536e0efccc] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.6 K 2024-11-12T12:25:37,337 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71b154f396284ec0b2c77d640187af68, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731414335235 2024-11-12T12:25:37,337 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd73b9851774bb485d6f411dd775ae6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731414335404 2024-11-12T12:25:37,337 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1d30acb41214e80a85dcd536e0efccc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731414335654 2024-11-12T12:25:37,348 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:37,349 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/2edaf36435194a6e9941b33350ecda48 is 50, key is test_row_0/C:col10/1731414336802/Put/seqid=0 2024-11-12T12:25:37,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742127_1303 (size=13017) 2024-11-12T12:25:37,384 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/2edaf36435194a6e9941b33350ecda48 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/2edaf36435194a6e9941b33350ecda48 2024-11-12T12:25:37,392 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/C of d1a5c014bc05d48b1ac89d4e1685996f into 2edaf36435194a6e9941b33350ecda48(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:37,392 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:37,392 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/C, priority=13, startTime=1731414337273; duration=0sec 2024-11-12T12:25:37,392 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:37,392 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:C 2024-11-12T12:25:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:37,403 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:37,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:25:37,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/a81e5f49c08f4d17b695971ad777a4c1 is 50, key is test_row_0/A:col10/1731414336824/Put/seqid=0 2024-11-12T12:25:37,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742128_1304 (size=14741) 2024-11-12T12:25:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414397475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414397475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414397476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414397476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414397481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T12:25:37,505 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:37,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-12T12:25:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T12:25:37,508 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:37,509 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:37,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:37,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414397591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414397591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414397592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414397592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414397592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T12:25:37,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T12:25:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:37,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:37,738 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/0b38d74d94664f35a8791bf0cc0ed39b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/0b38d74d94664f35a8791bf0cc0ed39b 2024-11-12T12:25:37,742 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/B of d1a5c014bc05d48b1ac89d4e1685996f into 0b38d74d94664f35a8791bf0cc0ed39b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:37,742 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:37,742 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/B, priority=13, startTime=1731414337273; duration=0sec 2024-11-12T12:25:37,743 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:37,743 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:B 2024-11-12T12:25:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414397800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414397801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414397801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414397801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414397801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:37,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T12:25:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:37,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T12:25:37,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/a81e5f49c08f4d17b695971ad777a4c1 2024-11-12T12:25:37,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/9e9d2e1984124ea5af06a46111bd5e7b is 50, key is test_row_0/B:col10/1731414336824/Put/seqid=0 2024-11-12T12:25:37,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742129_1305 (size=12301) 2024-11-12T12:25:37,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T12:25:37,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:37,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:37,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:37,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:37,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:38,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414398112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414398112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414398112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414398112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414398112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T12:25:38,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:38,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:38,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:38,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:38,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T12:25:38,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/9e9d2e1984124ea5af06a46111bd5e7b 2024-11-12T12:25:38,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/bc14ae95be6542b7a50ee777474e4707 is 50, key is test_row_0/C:col10/1731414336824/Put/seqid=0 2024-11-12T12:25:38,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742130_1306 (size=12301) 2024-11-12T12:25:38,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/bc14ae95be6542b7a50ee777474e4707 2024-11-12T12:25:38,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/a81e5f49c08f4d17b695971ad777a4c1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a81e5f49c08f4d17b695971ad777a4c1 2024-11-12T12:25:38,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T12:25:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:38,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:38,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:38,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a81e5f49c08f4d17b695971ad777a4c1, entries=200, sequenceid=323, filesize=14.4 K 2024-11-12T12:25:38,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/9e9d2e1984124ea5af06a46111bd5e7b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9e9d2e1984124ea5af06a46111bd5e7b 2024-11-12T12:25:38,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9e9d2e1984124ea5af06a46111bd5e7b, entries=150, sequenceid=323, filesize=12.0 K 2024-11-12T12:25:38,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/bc14ae95be6542b7a50ee777474e4707 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bc14ae95be6542b7a50ee777474e4707 2024-11-12T12:25:38,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bc14ae95be6542b7a50ee777474e4707, entries=150, sequenceid=323, filesize=12.0 K 2024-11-12T12:25:38,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d1a5c014bc05d48b1ac89d4e1685996f in 890ms, sequenceid=323, compaction requested=false 2024-11-12T12:25:38,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:38,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T12:25:38,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:38,430 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:25:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/0a5a41bfcc3a43b5b7bb9676ec9e27d1 is 50, key is test_row_0/A:col10/1731414337446/Put/seqid=0 2024-11-12T12:25:38,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742131_1307 (size=12301) 2024-11-12T12:25:38,439 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/0a5a41bfcc3a43b5b7bb9676ec9e27d1 2024-11-12T12:25:38,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/f8b36313bdb0422a9adfa8eb35ba5d67 is 50, key is test_row_0/B:col10/1731414337446/Put/seqid=0 2024-11-12T12:25:38,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742132_1308 (size=12301) 2024-11-12T12:25:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:38,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:38,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414398627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414398630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414398631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414398632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414398632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T12:25:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414398741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414398742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414398742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414398742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414398742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,856 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/f8b36313bdb0422a9adfa8eb35ba5d67 2024-11-12T12:25:38,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/4a3785cbf37e4388b5513e6a5aa710f3 is 50, key is test_row_0/C:col10/1731414337446/Put/seqid=0 2024-11-12T12:25:38,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742133_1309 (size=12301) 2024-11-12T12:25:38,914 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/4a3785cbf37e4388b5513e6a5aa710f3 2024-11-12T12:25:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/0a5a41bfcc3a43b5b7bb9676ec9e27d1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5a41bfcc3a43b5b7bb9676ec9e27d1 2024-11-12T12:25:38,922 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5a41bfcc3a43b5b7bb9676ec9e27d1, entries=150, sequenceid=349, filesize=12.0 K 2024-11-12T12:25:38,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/f8b36313bdb0422a9adfa8eb35ba5d67 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f8b36313bdb0422a9adfa8eb35ba5d67 2024-11-12T12:25:38,929 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f8b36313bdb0422a9adfa8eb35ba5d67, entries=150, sequenceid=349, filesize=12.0 K 2024-11-12T12:25:38,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/4a3785cbf37e4388b5513e6a5aa710f3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4a3785cbf37e4388b5513e6a5aa710f3 2024-11-12T12:25:38,936 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4a3785cbf37e4388b5513e6a5aa710f3, entries=150, sequenceid=349, filesize=12.0 K 2024-11-12T12:25:38,937 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for d1a5c014bc05d48b1ac89d4e1685996f in 508ms, sequenceid=349, compaction requested=true 2024-11-12T12:25:38,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:38,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:38,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-12T12:25:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-12T12:25:38,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-12T12:25:38,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-11-12T12:25:38,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.4390 sec 2024-11-12T12:25:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:38,955 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:38,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-12T12:25:38,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/233c74696e8748a98b127bf7ee45b851 is 50, key is test_row_0/A:col10/1731414338954/Put/seqid=0 2024-11-12T12:25:38,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742134_1310 (size=12301) 2024-11-12T12:25:38,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414398983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414398985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414398988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414398989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414398990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414399101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414399101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414399102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414399102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414399102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414399311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414399311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414399311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414399311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414399311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/233c74696e8748a98b127bf7ee45b851 2024-11-12T12:25:39,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/412d081b82bc49848c7443f7df971698 is 50, key is test_row_0/B:col10/1731414338954/Put/seqid=0 2024-11-12T12:25:39,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742135_1311 (size=12301) 2024-11-12T12:25:39,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414399621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414399621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414399621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414399621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414399621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T12:25:39,652 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:39,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-12T12:25:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T12:25:39,656 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:39,657 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:39,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T12:25:39,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/412d081b82bc49848c7443f7df971698 2024-11-12T12:25:39,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-12T12:25:39,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:39,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:39,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:39,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:39,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:39,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/7f7f836101a84534b2527b250affef29 is 50, key is test_row_0/C:col10/1731414338954/Put/seqid=0 2024-11-12T12:25:39,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742136_1312 (size=12301) 2024-11-12T12:25:39,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/7f7f836101a84534b2527b250affef29 2024-11-12T12:25:39,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/233c74696e8748a98b127bf7ee45b851 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/233c74696e8748a98b127bf7ee45b851 2024-11-12T12:25:39,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/233c74696e8748a98b127bf7ee45b851, entries=150, sequenceid=363, filesize=12.0 K 2024-11-12T12:25:39,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/412d081b82bc49848c7443f7df971698 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/412d081b82bc49848c7443f7df971698 2024-11-12T12:25:39,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/412d081b82bc49848c7443f7df971698, entries=150, sequenceid=363, filesize=12.0 K 2024-11-12T12:25:39,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/7f7f836101a84534b2527b250affef29 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7f7f836101a84534b2527b250affef29 2024-11-12T12:25:39,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7f7f836101a84534b2527b250affef29, entries=150, sequenceid=363, filesize=12.0 K 2024-11-12T12:25:39,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d1a5c014bc05d48b1ac89d4e1685996f in 895ms, sequenceid=363, compaction requested=true 2024-11-12T12:25:39,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:39,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:39,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:39,850 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:39,851 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:39,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:39,852 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:39,852 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/B is initiating minor compaction (all files) 2024-11-12T12:25:39,852 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/B in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:39,852 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/0b38d74d94664f35a8791bf0cc0ed39b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9e9d2e1984124ea5af06a46111bd5e7b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f8b36313bdb0422a9adfa8eb35ba5d67, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/412d081b82bc49848c7443f7df971698] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=48.8 K 2024-11-12T12:25:39,852 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:39,852 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/A is initiating minor compaction (all files) 2024-11-12T12:25:39,852 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/A in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:39,853 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/1c6037806a984bca91a281bde0e76041, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a81e5f49c08f4d17b695971ad777a4c1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5a41bfcc3a43b5b7bb9676ec9e27d1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/233c74696e8748a98b127bf7ee45b851] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=51.1 K 2024-11-12T12:25:39,854 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b38d74d94664f35a8791bf0cc0ed39b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731414335654 2024-11-12T12:25:39,854 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c6037806a984bca91a281bde0e76041, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731414335654 2024-11-12T12:25:39,855 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9d2e1984124ea5af06a46111bd5e7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731414336824 2024-11-12T12:25:39,855 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a81e5f49c08f4d17b695971ad777a4c1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731414336820 2024-11-12T12:25:39,855 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b36313bdb0422a9adfa8eb35ba5d67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731414337446 2024-11-12T12:25:39,855 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a5a41bfcc3a43b5b7bb9676ec9e27d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731414337446 2024-11-12T12:25:39,856 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 233c74696e8748a98b127bf7ee45b851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731414338630 2024-11-12T12:25:39,856 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 412d081b82bc49848c7443f7df971698, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731414338630 2024-11-12T12:25:39,868 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:39,869 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/e7e747930a2d4fcf9cf4a2fb3f190a3e is 50, key is test_row_0/B:col10/1731414338954/Put/seqid=0 2024-11-12T12:25:39,883 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#A#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:39,884 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/9d3527ee3dae4d3a828117b9b0d2cf0a is 50, key is test_row_0/A:col10/1731414338954/Put/seqid=0 2024-11-12T12:25:39,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742138_1314 (size=13153) 2024-11-12T12:25:39,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742137_1313 (size=13153) 2024-11-12T12:25:39,919 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/e7e747930a2d4fcf9cf4a2fb3f190a3e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/e7e747930a2d4fcf9cf4a2fb3f190a3e 2024-11-12T12:25:39,926 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/B of d1a5c014bc05d48b1ac89d4e1685996f into e7e747930a2d4fcf9cf4a2fb3f190a3e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:39,926 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:39,926 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/B, priority=12, startTime=1731414339851; duration=0sec 2024-11-12T12:25:39,926 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:39,926 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:B 2024-11-12T12:25:39,926 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:39,927 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:39,927 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/C is initiating minor compaction (all files) 2024-11-12T12:25:39,928 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/C in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:39,928 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/2edaf36435194a6e9941b33350ecda48, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bc14ae95be6542b7a50ee777474e4707, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4a3785cbf37e4388b5513e6a5aa710f3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7f7f836101a84534b2527b250affef29] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=48.8 K 2024-11-12T12:25:39,929 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2edaf36435194a6e9941b33350ecda48, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731414335654 2024-11-12T12:25:39,930 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting bc14ae95be6542b7a50ee777474e4707, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731414336824 2024-11-12T12:25:39,930 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3785cbf37e4388b5513e6a5aa710f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731414337446 2024-11-12T12:25:39,930 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f7f836101a84534b2527b250affef29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731414338630 2024-11-12T12:25:39,942 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:39,943 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/ceddd61997f846bc985d4b487d682456 is 50, key is test_row_0/C:col10/1731414338954/Put/seqid=0 2024-11-12T12:25:39,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-12T12:25:39,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:39,969 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-12T12:25:39,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742139_1315 (size=13153) 2024-11-12T12:25:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T12:25:39,978 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/ceddd61997f846bc985d4b487d682456 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ceddd61997f846bc985d4b487d682456 2024-11-12T12:25:39,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/6c5138deead04104a25d73b744f4fba6 is 50, key is test_row_0/A:col10/1731414338988/Put/seqid=0 2024-11-12T12:25:39,985 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/C of d1a5c014bc05d48b1ac89d4e1685996f into ceddd61997f846bc985d4b487d682456(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:39,985 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:39,985 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/C, priority=12, startTime=1731414339851; duration=0sec 2024-11-12T12:25:39,985 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:39,985 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:C 2024-11-12T12:25:39,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742140_1316 (size=12301) 2024-11-12T12:25:39,998 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/6c5138deead04104a25d73b744f4fba6 2024-11-12T12:25:40,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/96364dcda18f48749e003519f4d8b039 is 50, key is test_row_0/B:col10/1731414338988/Put/seqid=0 2024-11-12T12:25:40,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742141_1317 (size=12301) 2024-11-12T12:25:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:40,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. as already flushing 2024-11-12T12:25:40,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414400181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414400182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414400182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414400183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414400186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T12:25:40,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414400291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414400292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414400292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414400292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414400301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,310 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/9d3527ee3dae4d3a828117b9b0d2cf0a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9d3527ee3dae4d3a828117b9b0d2cf0a 2024-11-12T12:25:40,316 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/A of d1a5c014bc05d48b1ac89d4e1685996f into 9d3527ee3dae4d3a828117b9b0d2cf0a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:40,316 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:40,317 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/A, priority=12, startTime=1731414339850; duration=0sec 2024-11-12T12:25:40,317 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:40,317 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:A 2024-11-12T12:25:40,410 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/96364dcda18f48749e003519f4d8b039 2024-11-12T12:25:40,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/cf7701119e6d48ca97177bc6c015c79b is 50, key is test_row_0/C:col10/1731414338988/Put/seqid=0 2024-11-12T12:25:40,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742142_1318 (size=12301) 2024-11-12T12:25:40,444 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/cf7701119e6d48ca97177bc6c015c79b 2024-11-12T12:25:40,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/6c5138deead04104a25d73b744f4fba6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/6c5138deead04104a25d73b744f4fba6 2024-11-12T12:25:40,455 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/6c5138deead04104a25d73b744f4fba6, entries=150, sequenceid=386, filesize=12.0 K 2024-11-12T12:25:40,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/96364dcda18f48749e003519f4d8b039 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96364dcda18f48749e003519f4d8b039 2024-11-12T12:25:40,460 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96364dcda18f48749e003519f4d8b039, entries=150, sequenceid=386, filesize=12.0 K 2024-11-12T12:25:40,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/cf7701119e6d48ca97177bc6c015c79b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cf7701119e6d48ca97177bc6c015c79b 2024-11-12T12:25:40,466 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cf7701119e6d48ca97177bc6c015c79b, entries=150, sequenceid=386, filesize=12.0 K 2024-11-12T12:25:40,467 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d1a5c014bc05d48b1ac89d4e1685996f in 498ms, sequenceid=386, compaction requested=false 2024-11-12T12:25:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-12T12:25:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-12T12:25:40,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-12T12:25:40,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 812 msec 2024-11-12T12:25:40,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 820 msec 2024-11-12T12:25:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:40,503 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:40,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-12T12:25:40,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/7605141eec214d61ab70f739f4c7a3e8 is 50, key is test_row_0/A:col10/1731414340181/Put/seqid=0 2024-11-12T12:25:40,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742143_1319 (size=12301) 2024-11-12T12:25:40,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/7605141eec214d61ab70f739f4c7a3e8 2024-11-12T12:25:40,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/cc25455fde61488e98a0afdf2a6a1fce is 50, key is test_row_0/B:col10/1731414340181/Put/seqid=0 2024-11-12T12:25:40,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414400536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414400536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742144_1320 (size=12301) 2024-11-12T12:25:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414400543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414400544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414400545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414400651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414400651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414400651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414400651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414400651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,669 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-12T12:25:40,669 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:40,669 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:40,670 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,672 INFO [Thread-1118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1118. 2024-11-12T12:25:40,672 DEBUG [Thread-1118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:40,672 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,672 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,672 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-12T12:25:40,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:40,672 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:40,672 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,672 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,673 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-12T12:25:40,673 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:40,673 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,673 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:40,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:40,676 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-12T12:25:40,676 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:40,676 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,676 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:40,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T12:25:40,792 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:40,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414400861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414400861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414400861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414400862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414400862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:40,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/cc25455fde61488e98a0afdf2a6a1fce 2024-11-12T12:25:40,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/de20b0fd92cd449bb48c22c219d03cbd is 50, key is test_row_0/C:col10/1731414340181/Put/seqid=0 2024-11-12T12:25:40,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742145_1321 (size=12301) 2024-11-12T12:25:41,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:41,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42210 deadline: 1731414401171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:41,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42282 deadline: 1731414401171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42278 deadline: 1731414401172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42260 deadline: 1731414401172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:41,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42272 deadline: 1731414401172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:25:41,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/de20b0fd92cd449bb48c22c219d03cbd 2024-11-12T12:25:41,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/7605141eec214d61ab70f739f4c7a3e8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7605141eec214d61ab70f739f4c7a3e8 2024-11-12T12:25:41,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7605141eec214d61ab70f739f4c7a3e8, entries=150, sequenceid=404, filesize=12.0 K 2024-11-12T12:25:41,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/cc25455fde61488e98a0afdf2a6a1fce as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/cc25455fde61488e98a0afdf2a6a1fce 2024-11-12T12:25:41,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/cc25455fde61488e98a0afdf2a6a1fce, entries=150, sequenceid=404, filesize=12.0 K 2024-11-12T12:25:41,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/de20b0fd92cd449bb48c22c219d03cbd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/de20b0fd92cd449bb48c22c219d03cbd 2024-11-12T12:25:41,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/de20b0fd92cd449bb48c22c219d03cbd, entries=150, sequenceid=404, filesize=12.0 K 2024-11-12T12:25:41,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d1a5c014bc05d48b1ac89d4e1685996f in 884ms, sequenceid=404, compaction requested=true 2024-11-12T12:25:41,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:41,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:41,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:41,387 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:41,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:41,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:41,387 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:41,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1a5c014bc05d48b1ac89d4e1685996f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:41,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:41,388 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:41,388 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:41,388 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/A is initiating minor compaction (all files) 2024-11-12T12:25:41,388 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/B is initiating minor compaction (all files) 2024-11-12T12:25:41,388 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/B in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:41,388 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/A in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:41,388 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9d3527ee3dae4d3a828117b9b0d2cf0a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/6c5138deead04104a25d73b744f4fba6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7605141eec214d61ab70f739f4c7a3e8] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.9 K 2024-11-12T12:25:41,388 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/e7e747930a2d4fcf9cf4a2fb3f190a3e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96364dcda18f48749e003519f4d8b039, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/cc25455fde61488e98a0afdf2a6a1fce] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.9 K 2024-11-12T12:25:41,389 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting e7e747930a2d4fcf9cf4a2fb3f190a3e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731414338630 2024-11-12T12:25:41,389 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d3527ee3dae4d3a828117b9b0d2cf0a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731414338630 2024-11-12T12:25:41,389 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c5138deead04104a25d73b744f4fba6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731414338987 2024-11-12T12:25:41,389 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 96364dcda18f48749e003519f4d8b039, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731414338987 2024-11-12T12:25:41,389 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7605141eec214d61ab70f739f4c7a3e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731414340181 2024-11-12T12:25:41,389 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting cc25455fde61488e98a0afdf2a6a1fce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731414340181 2024-11-12T12:25:41,395 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:41,395 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#A#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:41,396 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/7733c323cb624fd7afee1de670ab30b1 is 50, key is test_row_0/B:col10/1731414340181/Put/seqid=0 2024-11-12T12:25:41,396 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/c89d0e27a3a1429787e9410b56419df0 is 50, key is test_row_0/A:col10/1731414340181/Put/seqid=0 2024-11-12T12:25:41,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742146_1322 (size=13255) 2024-11-12T12:25:41,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742147_1323 (size=13255) 2024-11-12T12:25:41,407 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/7733c323cb624fd7afee1de670ab30b1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/7733c323cb624fd7afee1de670ab30b1 2024-11-12T12:25:41,411 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/B of d1a5c014bc05d48b1ac89d4e1685996f into 7733c323cb624fd7afee1de670ab30b1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:41,411 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:41,411 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/B, priority=13, startTime=1731414341387; duration=0sec 2024-11-12T12:25:41,411 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:41,411 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:B 2024-11-12T12:25:41,411 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:41,412 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:41,412 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): d1a5c014bc05d48b1ac89d4e1685996f/C is initiating minor compaction (all files) 2024-11-12T12:25:41,412 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1a5c014bc05d48b1ac89d4e1685996f/C in TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:41,412 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ceddd61997f846bc985d4b487d682456, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cf7701119e6d48ca97177bc6c015c79b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/de20b0fd92cd449bb48c22c219d03cbd] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp, totalSize=36.9 K 2024-11-12T12:25:41,412 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ceddd61997f846bc985d4b487d682456, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731414338630 2024-11-12T12:25:41,413 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting cf7701119e6d48ca97177bc6c015c79b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731414338987 2024-11-12T12:25:41,413 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting de20b0fd92cd449bb48c22c219d03cbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731414340181 2024-11-12T12:25:41,418 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1a5c014bc05d48b1ac89d4e1685996f#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:41,419 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/f6dd9d163a5d4024a4015e288d7b6bc5 is 50, key is test_row_0/C:col10/1731414340181/Put/seqid=0 2024-11-12T12:25:41,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742148_1324 (size=13255) 2024-11-12T12:25:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:41,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:41,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-12T12:25:41,683 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-12T12:25:41,683 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-12T12:25:41,684 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:41,684 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:41,684 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,684 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,684 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-12T12:25:41,684 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:41,684 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,684 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,684 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,684 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:41,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:41,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:41,685 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-12T12:25:41,685 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:41,685 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,685 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:41,686 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-12T12:25:41,686 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:25:41,686 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,686 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,686 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3744 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3709 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3668 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3740 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3713 2024-11-12T12:25:41,687 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T12:25:41,687 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T12:25:41,687 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T12:25:41,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:41,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:25:41,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T12:25:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:41,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/d1c340de5c7848999d9e107667540fac is 50, key is test_row_0/A:col10/1731414341682/Put/seqid=0 2024-11-12T12:25:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-12T12:25:41,692 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414341692"}]},"ts":"1731414341692"} 2024-11-12T12:25:41,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742149_1325 (size=12301) 2024-11-12T12:25:41,694 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T12:25:41,694 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T12:25:41,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T12:25:41,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1a5c014bc05d48b1ac89d4e1685996f, UNASSIGN}] 2024-11-12T12:25:41,696 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1a5c014bc05d48b1ac89d4e1685996f, UNASSIGN 2024-11-12T12:25:41,697 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=d1a5c014bc05d48b1ac89d4e1685996f, regionState=CLOSING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:41,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1a5c014bc05d48b1ac89d4e1685996f, UNASSIGN because future has completed 2024-11-12T12:25:41,699 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T12:25:41,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:25:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-12T12:25:41,808 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/c89d0e27a3a1429787e9410b56419df0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/c89d0e27a3a1429787e9410b56419df0 2024-11-12T12:25:41,813 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/A of d1a5c014bc05d48b1ac89d4e1685996f into c89d0e27a3a1429787e9410b56419df0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:41,813 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:41,813 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/A, priority=13, startTime=1731414341387; duration=0sec 2024-11-12T12:25:41,813 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:41,813 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:A 2024-11-12T12:25:41,826 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/f6dd9d163a5d4024a4015e288d7b6bc5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/f6dd9d163a5d4024a4015e288d7b6bc5 2024-11-12T12:25:41,831 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1a5c014bc05d48b1ac89d4e1685996f/C of d1a5c014bc05d48b1ac89d4e1685996f into f6dd9d163a5d4024a4015e288d7b6bc5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:41,831 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:41,831 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., storeName=d1a5c014bc05d48b1ac89d4e1685996f/C, priority=13, startTime=1731414341387; duration=0sec 2024-11-12T12:25:41,831 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:41,831 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1a5c014bc05d48b1ac89d4e1685996f:C 2024-11-12T12:25:41,852 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:41,852 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T12:25:41,852 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing d1a5c014bc05d48b1ac89d4e1685996f, disabling compactions & flushes 2024-11-12T12:25:41,853 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-12T12:25:42,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/d1c340de5c7848999d9e107667540fac 2024-11-12T12:25:42,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/0dcf6a2716bb4d8ea3e527ac0d205150 is 50, key is test_row_0/B:col10/1731414341682/Put/seqid=0 2024-11-12T12:25:42,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742150_1326 (size=12301) 2024-11-12T12:25:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-12T12:25:42,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/0dcf6a2716bb4d8ea3e527ac0d205150 2024-11-12T12:25:42,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/4ce35d1c286141f4bd717dfef4270e5b is 50, key is test_row_0/C:col10/1731414341682/Put/seqid=0 2024-11-12T12:25:42,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742151_1327 (size=12301) 2024-11-12T12:25:42,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/4ce35d1c286141f4bd717dfef4270e5b 2024-11-12T12:25:42,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/d1c340de5c7848999d9e107667540fac as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1c340de5c7848999d9e107667540fac 2024-11-12T12:25:42,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1c340de5c7848999d9e107667540fac, entries=150, sequenceid=429, filesize=12.0 K 2024-11-12T12:25:42,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/0dcf6a2716bb4d8ea3e527ac0d205150 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/0dcf6a2716bb4d8ea3e527ac0d205150 2024-11-12T12:25:42,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/0dcf6a2716bb4d8ea3e527ac0d205150, entries=150, sequenceid=429, filesize=12.0 K 2024-11-12T12:25:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/4ce35d1c286141f4bd717dfef4270e5b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4ce35d1c286141f4bd717dfef4270e5b 2024-11-12T12:25:42,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4ce35d1c286141f4bd717dfef4270e5b, entries=150, sequenceid=429, filesize=12.0 K 2024-11-12T12:25:42,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for d1a5c014bc05d48b1ac89d4e1685996f in 857ms, sequenceid=429, compaction requested=false 2024-11-12T12:25:42,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1a5c014bc05d48b1ac89d4e1685996f: 2024-11-12T12:25:42,540 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:42,541 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:42,541 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. after waiting 0 ms 2024-11-12T12:25:42,541 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:42,541 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing d1a5c014bc05d48b1ac89d4e1685996f 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-12T12:25:42,547 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/d24334fe4bff499f9b574dc301788814 is 50, key is test_row_2/A:col10/1731414341685/Put/seqid=0 2024-11-12T12:25:42,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742152_1328 (size=7415) 2024-11-12T12:25:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-12T12:25:42,956 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/d24334fe4bff499f9b574dc301788814 2024-11-12T12:25:42,962 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/3b024f1ed54b444093e9de83e8a72b9d is 50, key is test_row_2/B:col10/1731414341685/Put/seqid=0 2024-11-12T12:25:42,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742153_1329 (size=7415) 2024-11-12T12:25:43,372 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/3b024f1ed54b444093e9de83e8a72b9d 2024-11-12T12:25:43,378 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/b1ac9d7f9f864e08967d7748b9fcf899 is 50, key is test_row_2/C:col10/1731414341685/Put/seqid=0 2024-11-12T12:25:43,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742154_1330 (size=7415) 2024-11-12T12:25:43,387 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/b1ac9d7f9f864e08967d7748b9fcf899 2024-11-12T12:25:43,392 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/A/d24334fe4bff499f9b574dc301788814 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d24334fe4bff499f9b574dc301788814 2024-11-12T12:25:43,395 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d24334fe4bff499f9b574dc301788814, entries=50, sequenceid=435, filesize=7.2 K 2024-11-12T12:25:43,396 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/B/3b024f1ed54b444093e9de83e8a72b9d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/3b024f1ed54b444093e9de83e8a72b9d 2024-11-12T12:25:43,400 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/3b024f1ed54b444093e9de83e8a72b9d, entries=50, sequenceid=435, filesize=7.2 K 2024-11-12T12:25:43,401 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/.tmp/C/b1ac9d7f9f864e08967d7748b9fcf899 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/b1ac9d7f9f864e08967d7748b9fcf899 2024-11-12T12:25:43,404 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/b1ac9d7f9f864e08967d7748b9fcf899, entries=50, sequenceid=435, filesize=7.2 K 2024-11-12T12:25:43,405 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for d1a5c014bc05d48b1ac89d4e1685996f in 864ms, sequenceid=435, compaction requested=true 2024-11-12T12:25:43,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/838d4726ea7a49f6b9d64366651f7c5e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5c760b1e7144d9b6ae260d4070f334, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/c9ef1107952d4142ae600bdcd3c76c3b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9fb6987ac63d446583c310156ea7ec5b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cd4a0a7fcad34a6a979526b0edb44172, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5f6f5dc0b2aa401498fb08498ab25c69, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/fa213122b98e4349972edf41d1f24bea, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/85cc9556ef05463c856d7f280f668e25, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/92701e36a4c34b559fbf23cdd6b39273, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a4fb504fbe91472196c34c65aba6771d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5b03ba80fad54c2e99b73ba18f6f62ae, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/40ac58fb634148e1876284caaf368bf0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cc17dd929fba4d5da7cc205ab43b2531, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0671abb8642c434ca92483d9f91fcf17, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/19624aa0070a453cafdfdc48eb098cb4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/431ca5e307634f61a5948ed0d68aa4ed, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7a0cf6f6a9954a33a324b839058bc803, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/71873c3153d047cc928ef5d0aea0edab, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/f860230330cf423b822c5ed2e803e602, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5766b4ef55d9450fb820dd0f2f67ddfa, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/1c6037806a984bca91a281bde0e76041, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1809ce46a6e47fa9cf25929ba98496d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a81e5f49c08f4d17b695971ad777a4c1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5a41bfcc3a43b5b7bb9676ec9e27d1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9d3527ee3dae4d3a828117b9b0d2cf0a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/233c74696e8748a98b127bf7ee45b851, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/6c5138deead04104a25d73b744f4fba6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7605141eec214d61ab70f739f4c7a3e8] to archive 2024-11-12T12:25:43,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:25:43,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/838d4726ea7a49f6b9d64366651f7c5e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/838d4726ea7a49f6b9d64366651f7c5e 2024-11-12T12:25:43,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5c760b1e7144d9b6ae260d4070f334 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5c760b1e7144d9b6ae260d4070f334 2024-11-12T12:25:43,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/c9ef1107952d4142ae600bdcd3c76c3b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/c9ef1107952d4142ae600bdcd3c76c3b 2024-11-12T12:25:43,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9fb6987ac63d446583c310156ea7ec5b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9fb6987ac63d446583c310156ea7ec5b 2024-11-12T12:25:43,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cd4a0a7fcad34a6a979526b0edb44172 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cd4a0a7fcad34a6a979526b0edb44172 2024-11-12T12:25:43,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5f6f5dc0b2aa401498fb08498ab25c69 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5f6f5dc0b2aa401498fb08498ab25c69 2024-11-12T12:25:43,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/fa213122b98e4349972edf41d1f24bea to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/fa213122b98e4349972edf41d1f24bea 2024-11-12T12:25:43,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/85cc9556ef05463c856d7f280f668e25 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/85cc9556ef05463c856d7f280f668e25 2024-11-12T12:25:43,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/92701e36a4c34b559fbf23cdd6b39273 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/92701e36a4c34b559fbf23cdd6b39273 2024-11-12T12:25:43,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a4fb504fbe91472196c34c65aba6771d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a4fb504fbe91472196c34c65aba6771d 2024-11-12T12:25:43,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5b03ba80fad54c2e99b73ba18f6f62ae to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5b03ba80fad54c2e99b73ba18f6f62ae 2024-11-12T12:25:43,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/40ac58fb634148e1876284caaf368bf0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/40ac58fb634148e1876284caaf368bf0 2024-11-12T12:25:43,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cc17dd929fba4d5da7cc205ab43b2531 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/cc17dd929fba4d5da7cc205ab43b2531 2024-11-12T12:25:43,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0671abb8642c434ca92483d9f91fcf17 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0671abb8642c434ca92483d9f91fcf17 2024-11-12T12:25:43,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/19624aa0070a453cafdfdc48eb098cb4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/19624aa0070a453cafdfdc48eb098cb4 2024-11-12T12:25:43,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/431ca5e307634f61a5948ed0d68aa4ed to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/431ca5e307634f61a5948ed0d68aa4ed 2024-11-12T12:25:43,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7a0cf6f6a9954a33a324b839058bc803 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7a0cf6f6a9954a33a324b839058bc803 2024-11-12T12:25:43,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/71873c3153d047cc928ef5d0aea0edab to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/71873c3153d047cc928ef5d0aea0edab 2024-11-12T12:25:43,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/f860230330cf423b822c5ed2e803e602 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/f860230330cf423b822c5ed2e803e602 2024-11-12T12:25:43,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5766b4ef55d9450fb820dd0f2f67ddfa to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/5766b4ef55d9450fb820dd0f2f67ddfa 2024-11-12T12:25:43,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/1c6037806a984bca91a281bde0e76041 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/1c6037806a984bca91a281bde0e76041 2024-11-12T12:25:43,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1809ce46a6e47fa9cf25929ba98496d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1809ce46a6e47fa9cf25929ba98496d 2024-11-12T12:25:43,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a81e5f49c08f4d17b695971ad777a4c1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/a81e5f49c08f4d17b695971ad777a4c1 2024-11-12T12:25:43,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5a41bfcc3a43b5b7bb9676ec9e27d1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/0a5a41bfcc3a43b5b7bb9676ec9e27d1 2024-11-12T12:25:43,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9d3527ee3dae4d3a828117b9b0d2cf0a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/9d3527ee3dae4d3a828117b9b0d2cf0a 2024-11-12T12:25:43,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/233c74696e8748a98b127bf7ee45b851 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/233c74696e8748a98b127bf7ee45b851 2024-11-12T12:25:43,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/6c5138deead04104a25d73b744f4fba6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/6c5138deead04104a25d73b744f4fba6 2024-11-12T12:25:43,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7605141eec214d61ab70f739f4c7a3e8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/7605141eec214d61ab70f739f4c7a3e8 2024-11-12T12:25:43,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4ca0752603284bb9bd1da2b987c6e834, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96b2828b84d34f0dbf1f4e9c7307af42, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2cbee7930239440299cb137a5af94beb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/c2235face2db47718c96291cb534b594, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2e2c1b475bcc400baa8579fe2eaf1dca, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/77e3cbf6c6ca4529b32e5c0d1f1f9da7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9a8c443550d048249a6c010ea54945e4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b9a116a86e474189a39b490d4fb8dab9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4541547028c0409e9ae7129220dbe0f8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/467407d86ffa44a891fbc9719927c58e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/dc50ad5375c94206bc535fe4c76682f8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/3c54b1fd57014069b8b89ce530d79f30, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f27d39300f8b4559bf77b535a7dd0b1a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4089539f4918497ea09aeca19e4810ff, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/5b589af3833d4fdb93c0e90facfb3cf3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/ac58b1534961427488aa741e00924748, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/293d27442afd4b46a44d4eb224bc25dc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b941e8402cda488ba11465cab9265d3f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/8738d56428404851a24677c0a0ac59c1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/bba2b0fa42234d9f883b9a288e8d2780, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/0b38d74d94664f35a8791bf0cc0ed39b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4632180e0a7d44839bf9c1304f6dfd57, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9e9d2e1984124ea5af06a46111bd5e7b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f8b36313bdb0422a9adfa8eb35ba5d67, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/e7e747930a2d4fcf9cf4a2fb3f190a3e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/412d081b82bc49848c7443f7df971698, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96364dcda18f48749e003519f4d8b039, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/cc25455fde61488e98a0afdf2a6a1fce] to archive 2024-11-12T12:25:43,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:25:43,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4ca0752603284bb9bd1da2b987c6e834 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4ca0752603284bb9bd1da2b987c6e834 2024-11-12T12:25:43,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96b2828b84d34f0dbf1f4e9c7307af42 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96b2828b84d34f0dbf1f4e9c7307af42 2024-11-12T12:25:43,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2cbee7930239440299cb137a5af94beb to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2cbee7930239440299cb137a5af94beb 2024-11-12T12:25:43,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/c2235face2db47718c96291cb534b594 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/c2235face2db47718c96291cb534b594 2024-11-12T12:25:43,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2e2c1b475bcc400baa8579fe2eaf1dca to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/2e2c1b475bcc400baa8579fe2eaf1dca 2024-11-12T12:25:43,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/77e3cbf6c6ca4529b32e5c0d1f1f9da7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/77e3cbf6c6ca4529b32e5c0d1f1f9da7 2024-11-12T12:25:43,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9a8c443550d048249a6c010ea54945e4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9a8c443550d048249a6c010ea54945e4 2024-11-12T12:25:43,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b9a116a86e474189a39b490d4fb8dab9 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b9a116a86e474189a39b490d4fb8dab9 2024-11-12T12:25:43,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4541547028c0409e9ae7129220dbe0f8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4541547028c0409e9ae7129220dbe0f8 2024-11-12T12:25:43,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/467407d86ffa44a891fbc9719927c58e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/467407d86ffa44a891fbc9719927c58e 2024-11-12T12:25:43,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/dc50ad5375c94206bc535fe4c76682f8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/dc50ad5375c94206bc535fe4c76682f8 2024-11-12T12:25:43,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/3c54b1fd57014069b8b89ce530d79f30 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/3c54b1fd57014069b8b89ce530d79f30 2024-11-12T12:25:43,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f27d39300f8b4559bf77b535a7dd0b1a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f27d39300f8b4559bf77b535a7dd0b1a 2024-11-12T12:25:43,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4089539f4918497ea09aeca19e4810ff to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4089539f4918497ea09aeca19e4810ff 2024-11-12T12:25:43,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/5b589af3833d4fdb93c0e90facfb3cf3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/5b589af3833d4fdb93c0e90facfb3cf3 2024-11-12T12:25:43,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/ac58b1534961427488aa741e00924748 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/ac58b1534961427488aa741e00924748 2024-11-12T12:25:43,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/293d27442afd4b46a44d4eb224bc25dc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/293d27442afd4b46a44d4eb224bc25dc 2024-11-12T12:25:43,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b941e8402cda488ba11465cab9265d3f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/b941e8402cda488ba11465cab9265d3f 2024-11-12T12:25:43,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/8738d56428404851a24677c0a0ac59c1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/8738d56428404851a24677c0a0ac59c1 2024-11-12T12:25:43,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/bba2b0fa42234d9f883b9a288e8d2780 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/bba2b0fa42234d9f883b9a288e8d2780 2024-11-12T12:25:43,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/0b38d74d94664f35a8791bf0cc0ed39b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/0b38d74d94664f35a8791bf0cc0ed39b 2024-11-12T12:25:43,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4632180e0a7d44839bf9c1304f6dfd57 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/4632180e0a7d44839bf9c1304f6dfd57 2024-11-12T12:25:43,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9e9d2e1984124ea5af06a46111bd5e7b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/9e9d2e1984124ea5af06a46111bd5e7b 2024-11-12T12:25:43,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f8b36313bdb0422a9adfa8eb35ba5d67 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/f8b36313bdb0422a9adfa8eb35ba5d67 2024-11-12T12:25:43,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/e7e747930a2d4fcf9cf4a2fb3f190a3e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/e7e747930a2d4fcf9cf4a2fb3f190a3e 2024-11-12T12:25:43,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/412d081b82bc49848c7443f7df971698 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/412d081b82bc49848c7443f7df971698 2024-11-12T12:25:43,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96364dcda18f48749e003519f4d8b039 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/96364dcda18f48749e003519f4d8b039 2024-11-12T12:25:43,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/cc25455fde61488e98a0afdf2a6a1fce to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/cc25455fde61488e98a0afdf2a6a1fce 2024-11-12T12:25:43,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ce0e3de6907f48bf92ad6276d3a6a1fc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/12231e634602499282216bb17a4faf8b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/88b9520ff40840038ef3197be90bed8f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/917fcb2524fa412da821270930028913, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cd3cfad0fda74312a29c1f0a976eb2c3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/0fcf791592eb4be198b3a58b04848180, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/269ba313c0344a308238213005f46cae, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/1d5a51786d3a4b04991518cfc9327deb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4aebf4797e5e492c9f9799ba034f555c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cdc0637bc89440dcb64c81c48be6cc44, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/db0cd70b03ed43ba94a2db96a3b7646d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/fc6e0eef95064e57b96f7eff763e503d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d37098c5e1744db7a715a0c08ec61cbc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/225965fde5c74d41b306773924b3e9d5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bebba284494f435ea7ae163a0938f3a5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/dcbd85b318e649cc972366abc1f64de9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7e1629d55adc436f88a130b328f2b071, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/71b154f396284ec0b2c77d640187af68, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/afe239592c3f43db9381d0c4e73330a4, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/9bd73b9851774bb485d6f411dd775ae6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/2edaf36435194a6e9941b33350ecda48, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d1d30acb41214e80a85dcd536e0efccc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bc14ae95be6542b7a50ee777474e4707, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4a3785cbf37e4388b5513e6a5aa710f3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ceddd61997f846bc985d4b487d682456, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7f7f836101a84534b2527b250affef29, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cf7701119e6d48ca97177bc6c015c79b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/de20b0fd92cd449bb48c22c219d03cbd] to archive 2024-11-12T12:25:43,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:25:43,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ce0e3de6907f48bf92ad6276d3a6a1fc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ce0e3de6907f48bf92ad6276d3a6a1fc 2024-11-12T12:25:43,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/12231e634602499282216bb17a4faf8b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/12231e634602499282216bb17a4faf8b 2024-11-12T12:25:43,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/88b9520ff40840038ef3197be90bed8f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/88b9520ff40840038ef3197be90bed8f 2024-11-12T12:25:43,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/917fcb2524fa412da821270930028913 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/917fcb2524fa412da821270930028913 2024-11-12T12:25:43,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cd3cfad0fda74312a29c1f0a976eb2c3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cd3cfad0fda74312a29c1f0a976eb2c3 2024-11-12T12:25:43,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/0fcf791592eb4be198b3a58b04848180 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/0fcf791592eb4be198b3a58b04848180 2024-11-12T12:25:43,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/269ba313c0344a308238213005f46cae to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/269ba313c0344a308238213005f46cae 2024-11-12T12:25:43,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/1d5a51786d3a4b04991518cfc9327deb to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/1d5a51786d3a4b04991518cfc9327deb 2024-11-12T12:25:43,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4aebf4797e5e492c9f9799ba034f555c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4aebf4797e5e492c9f9799ba034f555c 2024-11-12T12:25:43,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cdc0637bc89440dcb64c81c48be6cc44 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cdc0637bc89440dcb64c81c48be6cc44 2024-11-12T12:25:43,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/db0cd70b03ed43ba94a2db96a3b7646d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/db0cd70b03ed43ba94a2db96a3b7646d 2024-11-12T12:25:43,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/fc6e0eef95064e57b96f7eff763e503d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/fc6e0eef95064e57b96f7eff763e503d 2024-11-12T12:25:43,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d37098c5e1744db7a715a0c08ec61cbc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d37098c5e1744db7a715a0c08ec61cbc 2024-11-12T12:25:43,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/225965fde5c74d41b306773924b3e9d5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/225965fde5c74d41b306773924b3e9d5 2024-11-12T12:25:43,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bebba284494f435ea7ae163a0938f3a5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bebba284494f435ea7ae163a0938f3a5 2024-11-12T12:25:43,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/dcbd85b318e649cc972366abc1f64de9 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/dcbd85b318e649cc972366abc1f64de9 2024-11-12T12:25:43,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7e1629d55adc436f88a130b328f2b071 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7e1629d55adc436f88a130b328f2b071 2024-11-12T12:25:43,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/71b154f396284ec0b2c77d640187af68 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/71b154f396284ec0b2c77d640187af68 2024-11-12T12:25:43,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/afe239592c3f43db9381d0c4e73330a4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/afe239592c3f43db9381d0c4e73330a4 2024-11-12T12:25:43,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/9bd73b9851774bb485d6f411dd775ae6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/9bd73b9851774bb485d6f411dd775ae6 2024-11-12T12:25:43,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/2edaf36435194a6e9941b33350ecda48 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/2edaf36435194a6e9941b33350ecda48 2024-11-12T12:25:43,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d1d30acb41214e80a85dcd536e0efccc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/d1d30acb41214e80a85dcd536e0efccc 2024-11-12T12:25:43,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bc14ae95be6542b7a50ee777474e4707 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/bc14ae95be6542b7a50ee777474e4707 2024-11-12T12:25:43,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4a3785cbf37e4388b5513e6a5aa710f3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4a3785cbf37e4388b5513e6a5aa710f3 2024-11-12T12:25:43,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ceddd61997f846bc985d4b487d682456 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/ceddd61997f846bc985d4b487d682456 2024-11-12T12:25:43,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7f7f836101a84534b2527b250affef29 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/7f7f836101a84534b2527b250affef29 2024-11-12T12:25:43,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cf7701119e6d48ca97177bc6c015c79b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/cf7701119e6d48ca97177bc6c015c79b 2024-11-12T12:25:43,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/de20b0fd92cd449bb48c22c219d03cbd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/de20b0fd92cd449bb48c22c219d03cbd 2024-11-12T12:25:43,508 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/recovered.edits/438.seqid, newMaxSeqId=438, maxSeqId=1 2024-11-12T12:25:43,509 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. 2024-11-12T12:25:43,509 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for d1a5c014bc05d48b1ac89d4e1685996f: Waiting for close lock at 1731414341852Running coprocessor pre-close hooks at 1731414341852Disabling compacts and flushes for region at 1731414341852Disabling writes for close at 1731414342541 (+689 ms)Obtaining lock to block concurrent updates at 1731414342541Preparing flush snapshotting stores in d1a5c014bc05d48b1ac89d4e1685996f at 1731414342541Finished memstore snapshotting TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1731414342541Flushing stores of TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f. at 1731414342541Flushing d1a5c014bc05d48b1ac89d4e1685996f/A: creating writer at 1731414342542 (+1 ms)Flushing d1a5c014bc05d48b1ac89d4e1685996f/A: appending metadata at 1731414342546 (+4 ms)Flushing d1a5c014bc05d48b1ac89d4e1685996f/A: closing flushed file at 1731414342546Flushing d1a5c014bc05d48b1ac89d4e1685996f/B: creating writer at 1731414342959 (+413 ms)Flushing d1a5c014bc05d48b1ac89d4e1685996f/B: appending metadata at 1731414342961 (+2 ms)Flushing d1a5c014bc05d48b1ac89d4e1685996f/B: closing flushed file at 1731414342961Flushing d1a5c014bc05d48b1ac89d4e1685996f/C: creating writer at 1731414343375 (+414 ms)Flushing d1a5c014bc05d48b1ac89d4e1685996f/C: appending metadata at 1731414343377 (+2 ms)Flushing d1a5c014bc05d48b1ac89d4e1685996f/C: closing flushed file at 1731414343377Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69f6b3f2: reopening flushed file at 1731414343390 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9d04d57: reopening flushed file at 1731414343396 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28bde20f: reopening flushed file at 1731414343400 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for d1a5c014bc05d48b1ac89d4e1685996f in 864ms, sequenceid=435, compaction requested=true at 1731414343405 (+5 ms)Writing region close event to WAL at 1731414343506 (+101 ms)Running coprocessor post-close hooks at 1731414343509 (+3 ms)Closed at 1731414343509 2024-11-12T12:25:43,511 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:43,511 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=d1a5c014bc05d48b1ac89d4e1685996f, regionState=CLOSED 2024-11-12T12:25:43,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:25:43,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-12T12:25:43,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure d1a5c014bc05d48b1ac89d4e1685996f, server=4a91c05c96a4,38717,1731414257691 in 1.8140 sec 2024-11-12T12:25:43,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-12T12:25:43,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1a5c014bc05d48b1ac89d4e1685996f, UNASSIGN in 1.8200 sec 2024-11-12T12:25:43,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-12T12:25:43,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8230 sec 2024-11-12T12:25:43,519 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414343519"}]},"ts":"1731414343519"} 2024-11-12T12:25:43,520 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T12:25:43,520 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T12:25:43,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8330 sec 2024-11-12T12:25:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-12T12:25:43,841 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:43,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T12:25:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:43,844 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-12T12:25:43,845 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:43,847 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:43,850 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/recovered.edits] 2024-11-12T12:25:43,852 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/c89d0e27a3a1429787e9410b56419df0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/c89d0e27a3a1429787e9410b56419df0 2024-11-12T12:25:43,853 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1c340de5c7848999d9e107667540fac to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d1c340de5c7848999d9e107667540fac 2024-11-12T12:25:43,853 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d24334fe4bff499f9b574dc301788814 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/A/d24334fe4bff499f9b574dc301788814 2024-11-12T12:25:43,856 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/0dcf6a2716bb4d8ea3e527ac0d205150 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/0dcf6a2716bb4d8ea3e527ac0d205150 2024-11-12T12:25:43,858 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/3b024f1ed54b444093e9de83e8a72b9d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/3b024f1ed54b444093e9de83e8a72b9d 2024-11-12T12:25:43,859 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/7733c323cb624fd7afee1de670ab30b1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/B/7733c323cb624fd7afee1de670ab30b1 2024-11-12T12:25:43,861 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4ce35d1c286141f4bd717dfef4270e5b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/4ce35d1c286141f4bd717dfef4270e5b 2024-11-12T12:25:43,863 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/b1ac9d7f9f864e08967d7748b9fcf899 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/b1ac9d7f9f864e08967d7748b9fcf899 2024-11-12T12:25:43,865 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/f6dd9d163a5d4024a4015e288d7b6bc5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/C/f6dd9d163a5d4024a4015e288d7b6bc5 2024-11-12T12:25:43,868 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/recovered.edits/438.seqid to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f/recovered.edits/438.seqid 2024-11-12T12:25:43,869 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/d1a5c014bc05d48b1ac89d4e1685996f 2024-11-12T12:25:43,869 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T12:25:43,871 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:43,876 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T12:25:43,879 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T12:25:43,880 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:43,880 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T12:25:43,881 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731414343880"}]},"ts":"9223372036854775807"} 2024-11-12T12:25:43,883 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T12:25:43,883 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d1a5c014bc05d48b1ac89d4e1685996f, NAME => 'TestAcidGuarantees,,1731414318459.d1a5c014bc05d48b1ac89d4e1685996f.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T12:25:43,883 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T12:25:43,884 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731414343883"}]},"ts":"9223372036854775807"} 2024-11-12T12:25:43,886 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T12:25:43,887 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:43,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-12T12:25:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-12T12:25:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T12:25:43,954 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:43,960 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 241), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=569 (was 583), ProcessCount=11 (was 11), AvailableMemoryMB=6771 (was 6963) 2024-11-12T12:25:43,971 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=569, ProcessCount=11, AvailableMemoryMB=6771 2024-11-12T12:25:43,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T12:25:43,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:25:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:43,975 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T12:25:43,975 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:43,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-12T12:25:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T12:25:43,976 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T12:25:43,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742155_1331 (size=959) 2024-11-12T12:25:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T12:25:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T12:25:44,383 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a 2024-11-12T12:25:44,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742156_1332 (size=53) 2024-11-12T12:25:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T12:25:44,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:25:44,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 880bccb4d27e7b9d644e6b109920c175, disabling compactions & flushes 2024-11-12T12:25:44,789 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:44,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:44,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. after waiting 0 ms 2024-11-12T12:25:44,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:44,789 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:44,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 880bccb4d27e7b9d644e6b109920c175: Waiting for close lock at 1731414344789Disabling compacts and flushes for region at 1731414344789Disabling writes for close at 1731414344789Writing region close event to WAL at 1731414344789Closed at 1731414344789 2024-11-12T12:25:44,790 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T12:25:44,791 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731414344790"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731414344790"}]},"ts":"1731414344790"} 2024-11-12T12:25:44,792 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T12:25:44,792 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T12:25:44,793 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414344792"}]},"ts":"1731414344792"} 2024-11-12T12:25:44,794 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T12:25:44,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, ASSIGN}] 2024-11-12T12:25:44,795 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, ASSIGN 2024-11-12T12:25:44,796 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, ASSIGN; state=OFFLINE, location=4a91c05c96a4,38717,1731414257691; forceNewPlan=false, retain=false 2024-11-12T12:25:44,947 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=880bccb4d27e7b9d644e6b109920c175, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:44,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, ASSIGN because future has completed 2024-11-12T12:25:44,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:25:45,103 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:45,103 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:25:45,104 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,104 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:25:45,104 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,104 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,105 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,106 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880bccb4d27e7b9d644e6b109920c175 columnFamilyName A 2024-11-12T12:25:45,107 DEBUG [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:45,107 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(327): Store=880bccb4d27e7b9d644e6b109920c175/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:25:45,107 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,108 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880bccb4d27e7b9d644e6b109920c175 columnFamilyName B 2024-11-12T12:25:45,108 DEBUG [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:45,109 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(327): Store=880bccb4d27e7b9d644e6b109920c175/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:25:45,109 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,110 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880bccb4d27e7b9d644e6b109920c175 columnFamilyName C 2024-11-12T12:25:45,110 DEBUG [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:45,110 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(327): Store=880bccb4d27e7b9d644e6b109920c175/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:25:45,111 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T12:25:45,111 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,112 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,113 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,113 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,113 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:25:45,114 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,116 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T12:25:45,117 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 880bccb4d27e7b9d644e6b109920c175; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63140227, jitterRate=-0.05913729965686798}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:25:45,117 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:45,117 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 880bccb4d27e7b9d644e6b109920c175: Running coprocessor pre-open hook at 1731414345104Writing region info on filesystem at 1731414345104Initializing all the Stores at 1731414345105 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414345105Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414345105Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414345105Cleaning up temporary data from old regions at 1731414345113 (+8 ms)Running coprocessor post-open hooks at 1731414345117 (+4 ms)Region opened successfully at 1731414345117 2024-11-12T12:25:45,118 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., pid=93, masterSystemTime=1731414345100 2024-11-12T12:25:45,120 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:45,120 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:45,120 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=880bccb4d27e7b9d644e6b109920c175, regionState=OPEN, openSeqNum=2, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:45,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:25:45,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-12T12:25:45,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 in 173 msec 2024-11-12T12:25:45,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-12T12:25:45,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, ASSIGN in 330 msec 2024-11-12T12:25:45,126 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T12:25:45,126 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414345126"}]},"ts":"1731414345126"} 2024-11-12T12:25:45,127 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T12:25:45,128 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T12:25:45,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-12T12:25:46,046 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T12:25:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T12:25:46,121 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:46,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:46,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:46,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:46,123 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:46,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733569c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:46,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:46,125 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:46,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e531208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:46,125 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:46,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:46,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:46,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:46,127 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T12:25:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a91c05c96a4,41939,1731414256942 2024-11-12T12:25:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ebc4230 2024-11-12T12:25:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T12:25:46,129 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T12:25:46,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T12:25:46,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:25:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T12:25:46,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742157_1333 (size=995) 2024-11-12T12:25:46,541 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-12T12:25:46,541 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-12T12:25:46,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T12:25:46,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, REOPEN/MOVE}] 2024-11-12T12:25:46,547 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, REOPEN/MOVE 2024-11-12T12:25:46,548 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=880bccb4d27e7b9d644e6b109920c175, regionState=CLOSING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:46,548 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41939 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=4a91c05c96a4,38717,1731414257691, table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T12:25:46,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, REOPEN/MOVE because future has completed 2024-11-12T12:25:46,549 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T12:25:46,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:25:46,701 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:46,701 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T12:25:46,701 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 880bccb4d27e7b9d644e6b109920c175, disabling compactions & flushes 2024-11-12T12:25:46,701 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:46,701 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:46,702 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. after waiting 1 ms 2024-11-12T12:25:46,702 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:46,705 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T12:25:46,705 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:46,705 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 880bccb4d27e7b9d644e6b109920c175: Waiting for close lock at 1731414346701Running coprocessor pre-close hooks at 1731414346701Disabling compacts and flushes for region at 1731414346701Disabling writes for close at 1731414346702 (+1 ms)Writing region close event to WAL at 1731414346702Running coprocessor post-close hooks at 1731414346705 (+3 ms)Closed at 1731414346705 2024-11-12T12:25:46,705 WARN [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 880bccb4d27e7b9d644e6b109920c175 to self. 2024-11-12T12:25:46,707 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:46,707 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=880bccb4d27e7b9d644e6b109920c175, regionState=CLOSED 2024-11-12T12:25:46,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:25:46,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-12T12:25:46,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 in 160 msec 2024-11-12T12:25:46,711 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, REOPEN/MOVE; state=CLOSED, location=4a91c05c96a4,38717,1731414257691; forceNewPlan=false, retain=true 2024-11-12T12:25:46,862 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=880bccb4d27e7b9d644e6b109920c175, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:46,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, REOPEN/MOVE because future has completed 2024-11-12T12:25:46,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:25:47,017 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,017 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:25:47,018 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,018 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:25:47,018 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,018 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,019 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,020 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880bccb4d27e7b9d644e6b109920c175 columnFamilyName A 2024-11-12T12:25:47,021 DEBUG [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:47,021 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(327): Store=880bccb4d27e7b9d644e6b109920c175/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:25:47,021 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,022 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880bccb4d27e7b9d644e6b109920c175 columnFamilyName B 2024-11-12T12:25:47,022 DEBUG [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:47,022 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(327): Store=880bccb4d27e7b9d644e6b109920c175/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:25:47,022 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,023 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880bccb4d27e7b9d644e6b109920c175 columnFamilyName C 2024-11-12T12:25:47,023 DEBUG [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:47,023 INFO [StoreOpener-880bccb4d27e7b9d644e6b109920c175-1 {}] regionserver.HStore(327): Store=880bccb4d27e7b9d644e6b109920c175/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:25:47,023 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,023 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,024 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,025 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,025 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,025 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:25:47,026 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,027 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 880bccb4d27e7b9d644e6b109920c175; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69572949, jitterRate=0.0367177277803421}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:25:47,027 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,028 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 880bccb4d27e7b9d644e6b109920c175: Running coprocessor pre-open hook at 1731414347018Writing region info on filesystem at 1731414347018Initializing all the Stores at 1731414347018Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414347019 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414347019Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414347019Cleaning up temporary data from old regions at 1731414347025 (+6 ms)Running coprocessor post-open hooks at 1731414347027 (+2 ms)Region opened successfully at 1731414347028 (+1 ms) 2024-11-12T12:25:47,029 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., pid=98, masterSystemTime=1731414347015 2024-11-12T12:25:47,030 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,030 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=880bccb4d27e7b9d644e6b109920c175, regionState=OPEN, openSeqNum=5, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:25:47,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-12T12:25:47,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 in 169 msec 2024-11-12T12:25:47,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-12T12:25:47,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, REOPEN/MOVE in 488 msec 2024-11-12T12:25:47,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-12T12:25:47,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-12T12:25:47,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-12T12:25:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-12T12:25:47,040 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:47,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,042 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7253443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,043 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fca62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,045 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75af96d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,047 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@723a012f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,048 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56038aeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,049 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a24994a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,051 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300f645b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,052 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6258aa8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ea313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aff24e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,060 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffe6c0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,062 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b757ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,063 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,063 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,063 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4b5743, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,065 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@401291c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,067 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc5a01d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,069 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ba6348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,070 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,070 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efa9435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,072 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a706c5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,073 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:25:47,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:25:47,074 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:25:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:25:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:25:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368e259b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:25:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:25:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:25:47,076 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:25:47,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc14990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:25:47,077 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,077 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,078 DEBUG [Thread-1532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,078 DEBUG [Thread-1538 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-12T12:25:47,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:47,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,080 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:47,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T12:25:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,088 DEBUG [Thread-1523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,089 DEBUG [Thread-1530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,090 DEBUG [Thread-1521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,092 DEBUG [Thread-1519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,092 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,107 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:47,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:25:47,112 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,117 DEBUG [Thread-1534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:25:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:25:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:25:47,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:25:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:25:47,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f80311f8bca34a06ad1b3b806dfa1297_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414347107/Put/seqid=0 2024-11-12T12:25:47,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742158_1334 (size=17034) 2024-11-12T12:25:47,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414407145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414407153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414407154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55778 deadline: 1731414407159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414407160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T12:25:47,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:47,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:47,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414407271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414407271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414407272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55778 deadline: 1731414407281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414407281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T12:25:47,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414407481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414407481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414407481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55778 deadline: 1731414407491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414407491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,559 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:47,562 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f80311f8bca34a06ad1b3b806dfa1297_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f80311f8bca34a06ad1b3b806dfa1297_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:47,562 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/effdf7bba9cf4314bef5d6e29026b72c, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:47,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/effdf7bba9cf4314bef5d6e29026b72c is 175, key is test_row_0/A:col10/1731414347107/Put/seqid=0 2024-11-12T12:25:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742159_1335 (size=48139) 2024-11-12T12:25:47,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/effdf7bba9cf4314bef5d6e29026b72c 2024-11-12T12:25:47,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/c8432abd020b4f7fb2490e8fa34a940d is 50, key is test_row_0/B:col10/1731414347107/Put/seqid=0 2024-11-12T12:25:47,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742160_1336 (size=12001) 2024-11-12T12:25:47,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:47,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:47,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T12:25:47,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414407792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414407792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414407792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55778 deadline: 1731414407801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414407801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:47,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:47,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/c8432abd020b4f7fb2490e8fa34a940d 2024-11-12T12:25:48,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/3f43cd6c6533424db3c2dcba220d88a9 is 50, key is test_row_0/C:col10/1731414347107/Put/seqid=0 2024-11-12T12:25:48,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742161_1337 (size=12001) 2024-11-12T12:25:48,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:48,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:48,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T12:25:48,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:48,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414408301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414408301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:48,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414408311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:48,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414408311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55778 deadline: 1731414408321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:48,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/3f43cd6c6533424db3c2dcba220d88a9 2024-11-12T12:25:48,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/effdf7bba9cf4314bef5d6e29026b72c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/effdf7bba9cf4314bef5d6e29026b72c 2024-11-12T12:25:48,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/effdf7bba9cf4314bef5d6e29026b72c, entries=250, sequenceid=16, filesize=47.0 K 2024-11-12T12:25:48,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/c8432abd020b4f7fb2490e8fa34a940d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c8432abd020b4f7fb2490e8fa34a940d 2024-11-12T12:25:48,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c8432abd020b4f7fb2490e8fa34a940d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T12:25:48,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/3f43cd6c6533424db3c2dcba220d88a9 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3f43cd6c6533424db3c2dcba220d88a9 2024-11-12T12:25:48,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:48,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:48,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:48,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:48,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3f43cd6c6533424db3c2dcba220d88a9, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T12:25:48,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 880bccb4d27e7b9d644e6b109920c175 in 1355ms, sequenceid=16, compaction requested=false 2024-11-12T12:25:48,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:48,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T12:25:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:48,611 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:25:48,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e667540437754f4ea4feec7a37c394a5_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414347156/Put/seqid=0 2024-11-12T12:25:48,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742162_1338 (size=12154) 2024-11-12T12:25:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:48,636 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e667540437754f4ea4feec7a37c394a5_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e667540437754f4ea4feec7a37c394a5_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/3b46c98878e349f9b60c851778013f71, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/3b46c98878e349f9b60c851778013f71 is 175, key is test_row_0/A:col10/1731414347156/Put/seqid=0 2024-11-12T12:25:48,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742163_1339 (size=30955) 2024-11-12T12:25:48,653 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/3b46c98878e349f9b60c851778013f71 2024-11-12T12:25:48,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/366278c4907d4cda82db0be307fc0a4a is 50, key is test_row_0/B:col10/1731414347156/Put/seqid=0 2024-11-12T12:25:48,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742164_1340 (size=12001) 2024-11-12T12:25:49,043 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T12:25:49,067 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/366278c4907d4cda82db0be307fc0a4a 2024-11-12T12:25:49,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/ab8bd9de287247d08f3c0041f8800971 is 50, key is test_row_0/C:col10/1731414347156/Put/seqid=0 2024-11-12T12:25:49,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742165_1341 (size=12001) 2024-11-12T12:25:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T12:25:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:49,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:49,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414409337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414409338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55778 deadline: 1731414409341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,346 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414409345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414409345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414409451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414409451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414409462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414409462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,493 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/ab8bd9de287247d08f3c0041f8800971 2024-11-12T12:25:49,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/3b46c98878e349f9b60c851778013f71 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3b46c98878e349f9b60c851778013f71 2024-11-12T12:25:49,502 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3b46c98878e349f9b60c851778013f71, entries=150, sequenceid=41, filesize=30.2 K 2024-11-12T12:25:49,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/366278c4907d4cda82db0be307fc0a4a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/366278c4907d4cda82db0be307fc0a4a 2024-11-12T12:25:49,507 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/366278c4907d4cda82db0be307fc0a4a, entries=150, sequenceid=41, filesize=11.7 K 2024-11-12T12:25:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/ab8bd9de287247d08f3c0041f8800971 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ab8bd9de287247d08f3c0041f8800971 2024-11-12T12:25:49,513 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ab8bd9de287247d08f3c0041f8800971, entries=150, sequenceid=41, filesize=11.7 K 2024-11-12T12:25:49,514 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 880bccb4d27e7b9d644e6b109920c175 in 903ms, sequenceid=41, compaction requested=false 2024-11-12T12:25:49,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:49,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:49,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-12T12:25:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-12T12:25:49,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-12T12:25:49,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4350 sec 2024-11-12T12:25:49,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.4390 sec 2024-11-12T12:25:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:49,667 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:49,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-12T12:25:49,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b7b6a36734d647b091349d02da84bc65_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414349666/Put/seqid=0 2024-11-12T12:25:49,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742166_1342 (size=14594) 2024-11-12T12:25:49,685 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:49,689 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b7b6a36734d647b091349d02da84bc65_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b7b6a36734d647b091349d02da84bc65_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:49,690 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/631ef8191f084c66bdbd8ba4804799a6, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:49,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/631ef8191f084c66bdbd8ba4804799a6 is 175, key is test_row_0/A:col10/1731414349666/Put/seqid=0 2024-11-12T12:25:49,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742167_1343 (size=39549) 2024-11-12T12:25:49,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/631ef8191f084c66bdbd8ba4804799a6 2024-11-12T12:25:49,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/de23996fc1e342d092bf7a0ada786bdf is 50, key is test_row_0/B:col10/1731414349666/Put/seqid=0 2024-11-12T12:25:49,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742168_1344 (size=12001) 2024-11-12T12:25:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414409726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414409727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414409737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414409742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414409853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414409853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414409853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414409861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414410071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414410072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414410072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414410072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/de23996fc1e342d092bf7a0ada786bdf 2024-11-12T12:25:50,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/6b9bc791e94e47f3ab6cf3f67474c10b is 50, key is test_row_0/C:col10/1731414349666/Put/seqid=0 2024-11-12T12:25:50,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742169_1345 (size=12001) 2024-11-12T12:25:50,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414410381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414410382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414410382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414410382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/6b9bc791e94e47f3ab6cf3f67474c10b 2024-11-12T12:25:50,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/631ef8191f084c66bdbd8ba4804799a6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/631ef8191f084c66bdbd8ba4804799a6 2024-11-12T12:25:50,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/631ef8191f084c66bdbd8ba4804799a6, entries=200, sequenceid=53, filesize=38.6 K 2024-11-12T12:25:50,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/de23996fc1e342d092bf7a0ada786bdf as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/de23996fc1e342d092bf7a0ada786bdf 2024-11-12T12:25:50,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/de23996fc1e342d092bf7a0ada786bdf, entries=150, sequenceid=53, filesize=11.7 K 2024-11-12T12:25:50,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/6b9bc791e94e47f3ab6cf3f67474c10b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6b9bc791e94e47f3ab6cf3f67474c10b 2024-11-12T12:25:50,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6b9bc791e94e47f3ab6cf3f67474c10b, entries=150, sequenceid=53, filesize=11.7 K 2024-11-12T12:25:50,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 880bccb4d27e7b9d644e6b109920c175 in 888ms, sequenceid=53, compaction requested=true 2024-11-12T12:25:50,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:50,555 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:50,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:50,556 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:50,556 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/A is initiating minor compaction (all files) 2024-11-12T12:25:50,556 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/A in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:50,556 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/effdf7bba9cf4314bef5d6e29026b72c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3b46c98878e349f9b60c851778013f71, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/631ef8191f084c66bdbd8ba4804799a6] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=115.9 K 2024-11-12T12:25:50,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:50,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:50,556 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:50,556 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:50,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:50,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:50,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:50,556 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/effdf7bba9cf4314bef5d6e29026b72c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3b46c98878e349f9b60c851778013f71, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/631ef8191f084c66bdbd8ba4804799a6] 2024-11-12T12:25:50,557 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting effdf7bba9cf4314bef5d6e29026b72c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731414347101 2024-11-12T12:25:50,557 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b46c98878e349f9b60c851778013f71, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731414347135 2024-11-12T12:25:50,557 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:50,557 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/B is initiating minor compaction (all files) 2024-11-12T12:25:50,557 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/B in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:50,557 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c8432abd020b4f7fb2490e8fa34a940d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/366278c4907d4cda82db0be307fc0a4a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/de23996fc1e342d092bf7a0ada786bdf] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=35.2 K 2024-11-12T12:25:50,557 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631ef8191f084c66bdbd8ba4804799a6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731414349336 2024-11-12T12:25:50,557 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting c8432abd020b4f7fb2490e8fa34a940d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731414347101 2024-11-12T12:25:50,558 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 366278c4907d4cda82db0be307fc0a4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731414347135 2024-11-12T12:25:50,558 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting de23996fc1e342d092bf7a0ada786bdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731414349336 2024-11-12T12:25:50,562 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:50,563 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#B#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:50,564 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/d88c6d2b886e41ab84e875a064c14dc5 is 50, key is test_row_0/B:col10/1731414349666/Put/seqid=0 2024-11-12T12:25:50,569 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111203fe78d20e9041e88367170ce2f83614_880bccb4d27e7b9d644e6b109920c175 store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:50,570 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111203fe78d20e9041e88367170ce2f83614_880bccb4d27e7b9d644e6b109920c175, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:50,571 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111203fe78d20e9041e88367170ce2f83614_880bccb4d27e7b9d644e6b109920c175 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:50,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742170_1346 (size=12104) 2024-11-12T12:25:50,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742171_1347 (size=4469) 2024-11-12T12:25:50,586 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#A#compaction#294 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:50,586 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/978cf1f9b62a4485a7ba0d6d8e584c56 is 175, key is test_row_0/A:col10/1731414349666/Put/seqid=0 2024-11-12T12:25:50,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742172_1348 (size=31058) 2024-11-12T12:25:50,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:50,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:25:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:50,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111279e6352ef84845cfaa9dda14503342cd_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414349722/Put/seqid=0 2024-11-12T12:25:50,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414410924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742173_1349 (size=14594) 2024-11-12T12:25:50,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414410926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414410929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414410931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:50,983 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/d88c6d2b886e41ab84e875a064c14dc5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d88c6d2b886e41ab84e875a064c14dc5 2024-11-12T12:25:50,988 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/B of 880bccb4d27e7b9d644e6b109920c175 into d88c6d2b886e41ab84e875a064c14dc5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:50,988 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:50,988 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/B, priority=13, startTime=1731414350556; duration=0sec 2024-11-12T12:25:50,988 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:50,988 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:B 2024-11-12T12:25:50,989 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:50,992 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:50,993 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/C is initiating minor compaction (all files) 2024-11-12T12:25:50,993 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/C in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:50,993 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3f43cd6c6533424db3c2dcba220d88a9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ab8bd9de287247d08f3c0041f8800971, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6b9bc791e94e47f3ab6cf3f67474c10b] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=35.2 K 2024-11-12T12:25:50,993 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f43cd6c6533424db3c2dcba220d88a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731414347101 2024-11-12T12:25:50,993 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ab8bd9de287247d08f3c0041f8800971, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731414347135 2024-11-12T12:25:50,993 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b9bc791e94e47f3ab6cf3f67474c10b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731414349336 2024-11-12T12:25:51,006 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/978cf1f9b62a4485a7ba0d6d8e584c56 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/978cf1f9b62a4485a7ba0d6d8e584c56 2024-11-12T12:25:51,010 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/A of 880bccb4d27e7b9d644e6b109920c175 into 978cf1f9b62a4485a7ba0d6d8e584c56(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:51,010 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:51,010 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/A, priority=13, startTime=1731414350555; duration=0sec 2024-11-12T12:25:51,010 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:51,010 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:51,010 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:A 2024-11-12T12:25:51,011 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/4a1d217feec34001972d731c8d430d6a is 50, key is test_row_0/C:col10/1731414349666/Put/seqid=0 2024-11-12T12:25:51,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742174_1350 (size=12104) 2024-11-12T12:25:51,022 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/4a1d217feec34001972d731c8d430d6a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/4a1d217feec34001972d731c8d430d6a 2024-11-12T12:25:51,029 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/C of 880bccb4d27e7b9d644e6b109920c175 into 4a1d217feec34001972d731c8d430d6a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:51,029 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:51,029 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/C, priority=13, startTime=1731414350556; duration=0sec 2024-11-12T12:25:51,029 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:51,029 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:C 2024-11-12T12:25:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414411041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414411041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414411041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414411051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T12:25:51,251 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:51,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-12T12:25:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T12:25:51,254 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:51,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414411251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,254 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:51,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:51,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414411251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414411261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414411262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:51,333 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111279e6352ef84845cfaa9dda14503342cd_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111279e6352ef84845cfaa9dda14503342cd_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:51,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/36fe1497156d40628bfe1cdf70f9fda2, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:51,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/36fe1497156d40628bfe1cdf70f9fda2 is 175, key is test_row_0/A:col10/1731414349722/Put/seqid=0 2024-11-12T12:25:51,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742175_1351 (size=39549) 2024-11-12T12:25:51,339 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/36fe1497156d40628bfe1cdf70f9fda2 2024-11-12T12:25:51,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/286f80452a52406f9220c291e55a6293 is 50, key is test_row_0/B:col10/1731414349722/Put/seqid=0 2024-11-12T12:25:51,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742176_1352 (size=12001) 2024-11-12T12:25:51,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/286f80452a52406f9220c291e55a6293 2024-11-12T12:25:51,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/a7fbbc8992fc4332ab303b6aca3c95cb is 50, key is test_row_0/C:col10/1731414349722/Put/seqid=0 2024-11-12T12:25:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T12:25:51,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55778 deadline: 1731414411362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,363 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742177_1353 (size=12001) 2024-11-12T12:25:51,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/a7fbbc8992fc4332ab303b6aca3c95cb 2024-11-12T12:25:51,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/36fe1497156d40628bfe1cdf70f9fda2 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/36fe1497156d40628bfe1cdf70f9fda2 2024-11-12T12:25:51,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/36fe1497156d40628bfe1cdf70f9fda2, entries=200, sequenceid=78, filesize=38.6 K 2024-11-12T12:25:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/286f80452a52406f9220c291e55a6293 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/286f80452a52406f9220c291e55a6293 2024-11-12T12:25:51,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/286f80452a52406f9220c291e55a6293, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T12:25:51,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/a7fbbc8992fc4332ab303b6aca3c95cb as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7fbbc8992fc4332ab303b6aca3c95cb 2024-11-12T12:25:51,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7fbbc8992fc4332ab303b6aca3c95cb, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T12:25:51,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 880bccb4d27e7b9d644e6b109920c175 in 485ms, sequenceid=78, compaction requested=false 2024-11-12T12:25:51,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:51,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T12:25:51,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:51,408 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:25:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cd9d8793e0ca4ac9af84f2b1ce3f8cf4_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414350926/Put/seqid=0 2024-11-12T12:25:51,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742178_1354 (size=12154) 2024-11-12T12:25:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:51,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T12:25:51,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414411605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414411612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414411613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414411615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414411721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414411731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414411731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414411731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:51,834 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cd9d8793e0ca4ac9af84f2b1ce3f8cf4_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cd9d8793e0ca4ac9af84f2b1ce3f8cf4_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/1f4bbf8337174a75b8bb06604c6d6e11, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/1f4bbf8337174a75b8bb06604c6d6e11 is 175, key is test_row_0/A:col10/1731414350926/Put/seqid=0 2024-11-12T12:25:51,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742179_1355 (size=30955) 2024-11-12T12:25:51,844 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/1f4bbf8337174a75b8bb06604c6d6e11 2024-11-12T12:25:51,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/40ce0ede03a049f59f84cca00844fbc3 is 50, key is test_row_0/B:col10/1731414350926/Put/seqid=0 2024-11-12T12:25:51,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742180_1356 (size=12001) 2024-11-12T12:25:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T12:25:51,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414411932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414411941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414411941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414411941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414412241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,265 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/40ce0ede03a049f59f84cca00844fbc3 2024-11-12T12:25:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/eebaa21a08184320ba15bca973c2a246 is 50, key is test_row_0/C:col10/1731414350926/Put/seqid=0 2024-11-12T12:25:52,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414412261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414412261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414412261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742181_1357 (size=12001) 2024-11-12T12:25:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T12:25:52,693 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/eebaa21a08184320ba15bca973c2a246 2024-11-12T12:25:52,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/1f4bbf8337174a75b8bb06604c6d6e11 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/1f4bbf8337174a75b8bb06604c6d6e11 2024-11-12T12:25:52,708 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/1f4bbf8337174a75b8bb06604c6d6e11, entries=150, sequenceid=92, filesize=30.2 K 2024-11-12T12:25:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/40ce0ede03a049f59f84cca00844fbc3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/40ce0ede03a049f59f84cca00844fbc3 2024-11-12T12:25:52,713 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/40ce0ede03a049f59f84cca00844fbc3, entries=150, sequenceid=92, filesize=11.7 K 2024-11-12T12:25:52,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/eebaa21a08184320ba15bca973c2a246 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eebaa21a08184320ba15bca973c2a246 2024-11-12T12:25:52,717 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eebaa21a08184320ba15bca973c2a246, entries=150, sequenceid=92, filesize=11.7 K 2024-11-12T12:25:52,718 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 880bccb4d27e7b9d644e6b109920c175 in 1311ms, sequenceid=92, compaction requested=true 2024-11-12T12:25:52,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:52,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:52,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-12T12:25:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-12T12:25:52,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-12T12:25:52,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-11-12T12:25:52,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4710 sec 2024-11-12T12:25:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:52,769 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:52,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-12T12:25:52,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125ba56a26217f416d8b8d4089f95416c0_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414351603/Put/seqid=0 2024-11-12T12:25:52,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742182_1358 (size=14594) 2024-11-12T12:25:52,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414412795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414412808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414412809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414412809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414412911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414412931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414412931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414412931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414413121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414413151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414413151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414413151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,184 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:53,187 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125ba56a26217f416d8b8d4089f95416c0_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125ba56a26217f416d8b8d4089f95416c0_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:53,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/90f2f1c8d8d94bf4b49f6cb6acb4d033, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:53,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/90f2f1c8d8d94bf4b49f6cb6acb4d033 is 175, key is test_row_0/A:col10/1731414351603/Put/seqid=0 2024-11-12T12:25:53,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742183_1359 (size=39549) 2024-11-12T12:25:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T12:25:53,401 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:53,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-12T12:25:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T12:25:53,403 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:53,403 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:53,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414413431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414413461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414413471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414413471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T12:25:53,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T12:25:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:53,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:53,592 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/90f2f1c8d8d94bf4b49f6cb6acb4d033 2024-11-12T12:25:53,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/208cb68d697f47048a55f74f5d0ac9d9 is 50, key is test_row_0/B:col10/1731414351603/Put/seqid=0 2024-11-12T12:25:53,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742184_1360 (size=12001) 2024-11-12T12:25:53,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/208cb68d697f47048a55f74f5d0ac9d9 2024-11-12T12:25:53,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/566d0257af0b426fb8650d194f53217f is 50, key is test_row_0/C:col10/1731414351603/Put/seqid=0 2024-11-12T12:25:53,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742185_1361 (size=12001) 2024-11-12T12:25:53,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T12:25:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:53,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T12:25:53,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T12:25:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:53,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:53,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414413951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414413971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:53,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414413981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:54,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414413991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:54,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T12:25:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:54,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T12:25:54,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/566d0257af0b426fb8650d194f53217f 2024-11-12T12:25:54,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/90f2f1c8d8d94bf4b49f6cb6acb4d033 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/90f2f1c8d8d94bf4b49f6cb6acb4d033 2024-11-12T12:25:54,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/90f2f1c8d8d94bf4b49f6cb6acb4d033, entries=200, sequenceid=118, filesize=38.6 K 2024-11-12T12:25:54,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/208cb68d697f47048a55f74f5d0ac9d9 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/208cb68d697f47048a55f74f5d0ac9d9 2024-11-12T12:25:54,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/208cb68d697f47048a55f74f5d0ac9d9, entries=150, sequenceid=118, filesize=11.7 K 2024-11-12T12:25:54,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/566d0257af0b426fb8650d194f53217f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/566d0257af0b426fb8650d194f53217f 2024-11-12T12:25:54,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/566d0257af0b426fb8650d194f53217f, entries=150, sequenceid=118, filesize=11.7 K 2024-11-12T12:25:54,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 880bccb4d27e7b9d644e6b109920c175 in 1285ms, sequenceid=118, compaction requested=true 2024-11-12T12:25:54,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:54,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:54,054 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:54,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:54,054 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:54,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:54,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:54,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:54,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:54,056 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:54,056 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/A is initiating minor compaction (all files) 2024-11-12T12:25:54,056 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/A in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:54,056 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/978cf1f9b62a4485a7ba0d6d8e584c56, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/36fe1497156d40628bfe1cdf70f9fda2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/1f4bbf8337174a75b8bb06604c6d6e11, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/90f2f1c8d8d94bf4b49f6cb6acb4d033] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=137.8 K 2024-11-12T12:25:54,056 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:54,056 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/978cf1f9b62a4485a7ba0d6d8e584c56, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/36fe1497156d40628bfe1cdf70f9fda2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/1f4bbf8337174a75b8bb06604c6d6e11, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/90f2f1c8d8d94bf4b49f6cb6acb4d033] 2024-11-12T12:25:54,057 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:54,057 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/B is initiating minor compaction (all files) 2024-11-12T12:25:54,057 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/B in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:54,057 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d88c6d2b886e41ab84e875a064c14dc5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/286f80452a52406f9220c291e55a6293, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/40ce0ede03a049f59f84cca00844fbc3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/208cb68d697f47048a55f74f5d0ac9d9] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=47.0 K 2024-11-12T12:25:54,057 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d88c6d2b886e41ab84e875a064c14dc5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731414349336 2024-11-12T12:25:54,057 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 978cf1f9b62a4485a7ba0d6d8e584c56, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731414349336 2024-11-12T12:25:54,058 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 286f80452a52406f9220c291e55a6293, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731414349722 2024-11-12T12:25:54,058 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36fe1497156d40628bfe1cdf70f9fda2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731414349722 2024-11-12T12:25:54,058 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 40ce0ede03a049f59f84cca00844fbc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731414350911 2024-11-12T12:25:54,058 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f4bbf8337174a75b8bb06604c6d6e11, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731414350911 2024-11-12T12:25:54,059 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 208cb68d697f47048a55f74f5d0ac9d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731414351603 2024-11-12T12:25:54,059 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f2f1c8d8d94bf4b49f6cb6acb4d033, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731414351602 2024-11-12T12:25:54,065 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:54,074 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411129f84dc8cd36f4f64831733a388765a11_880bccb4d27e7b9d644e6b109920c175 store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:54,076 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#B#compaction#307 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:54,077 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/bf707c34387746ce8abea46d7fb3b23f is 50, key is test_row_0/B:col10/1731414351603/Put/seqid=0 2024-11-12T12:25:54,077 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411129f84dc8cd36f4f64831733a388765a11_880bccb4d27e7b9d644e6b109920c175, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:54,077 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129f84dc8cd36f4f64831733a388765a11_880bccb4d27e7b9d644e6b109920c175 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:54,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742187_1363 (size=4469) 2024-11-12T12:25:54,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742186_1362 (size=12241) 2024-11-12T12:25:54,081 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#A#compaction#306 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:54,081 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/64341b3ce6f54ebb8bce960c165ecbbc is 175, key is test_row_0/A:col10/1731414351603/Put/seqid=0 2024-11-12T12:25:54,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742188_1364 (size=31195) 2024-11-12T12:25:54,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T12:25:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:54,166 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-12T12:25:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111289aed92136bd49c9a958577af5e74d37_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414352788/Put/seqid=0 2024-11-12T12:25:54,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742189_1365 (size=12204) 2024-11-12T12:25:54,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,182 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111289aed92136bd49c9a958577af5e74d37_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111289aed92136bd49c9a958577af5e74d37_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/2fa8fddc847f42e4b9b2cd3349559768, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/2fa8fddc847f42e4b9b2cd3349559768 is 175, key is test_row_0/A:col10/1731414352788/Put/seqid=0 2024-11-12T12:25:54,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742190_1366 (size=31005) 2024-11-12T12:25:54,486 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/bf707c34387746ce8abea46d7fb3b23f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/bf707c34387746ce8abea46d7fb3b23f 2024-11-12T12:25:54,490 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/64341b3ce6f54ebb8bce960c165ecbbc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/64341b3ce6f54ebb8bce960c165ecbbc 2024-11-12T12:25:54,492 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/B of 880bccb4d27e7b9d644e6b109920c175 into bf707c34387746ce8abea46d7fb3b23f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:54,492 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:54,492 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/B, priority=12, startTime=1731414354054; duration=0sec 2024-11-12T12:25:54,492 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:54,492 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:B 2024-11-12T12:25:54,492 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:25:54,495 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:25:54,495 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/C is initiating minor compaction (all files) 2024-11-12T12:25:54,495 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/C in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:54,495 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/4a1d217feec34001972d731c8d430d6a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7fbbc8992fc4332ab303b6aca3c95cb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eebaa21a08184320ba15bca973c2a246, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/566d0257af0b426fb8650d194f53217f] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=47.0 K 2024-11-12T12:25:54,497 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a1d217feec34001972d731c8d430d6a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731414349336 2024-11-12T12:25:54,498 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a7fbbc8992fc4332ab303b6aca3c95cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731414349722 2024-11-12T12:25:54,498 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting eebaa21a08184320ba15bca973c2a246, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731414350911 2024-11-12T12:25:54,499 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 566d0257af0b426fb8650d194f53217f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731414351603 2024-11-12T12:25:54,500 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/A of 880bccb4d27e7b9d644e6b109920c175 into 64341b3ce6f54ebb8bce960c165ecbbc(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:54,500 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:54,500 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/A, priority=12, startTime=1731414354054; duration=0sec 2024-11-12T12:25:54,500 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:54,500 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:A 2024-11-12T12:25:54,520 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:54,520 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/52d606b6f652495288d91963a5e7179e is 50, key is test_row_0/C:col10/1731414351603/Put/seqid=0 2024-11-12T12:25:54,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742191_1367 (size=12241) 2024-11-12T12:25:54,532 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/52d606b6f652495288d91963a5e7179e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/52d606b6f652495288d91963a5e7179e 2024-11-12T12:25:54,539 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/C of 880bccb4d27e7b9d644e6b109920c175 into 52d606b6f652495288d91963a5e7179e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:54,539 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:54,539 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/C, priority=12, startTime=1731414354054; duration=0sec 2024-11-12T12:25:54,539 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:54,539 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:C 2024-11-12T12:25:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T12:25:54,587 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/2fa8fddc847f42e4b9b2cd3349559768 2024-11-12T12:25:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/289a78e6d940462fa6b12a8509dd164c is 50, key is test_row_0/B:col10/1731414352788/Put/seqid=0 2024-11-12T12:25:54,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742192_1368 (size=12051) 2024-11-12T12:25:54,618 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/289a78e6d940462fa6b12a8509dd164c 2024-11-12T12:25:54,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/d38d7535289b4f26b11d448b723ad694 is 50, key is test_row_0/C:col10/1731414352788/Put/seqid=0 2024-11-12T12:25:54,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742193_1369 (size=12051) 2024-11-12T12:25:54,650 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/d38d7535289b4f26b11d448b723ad694 2024-11-12T12:25:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/2fa8fddc847f42e4b9b2cd3349559768 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2fa8fddc847f42e4b9b2cd3349559768 2024-11-12T12:25:54,665 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2fa8fddc847f42e4b9b2cd3349559768, entries=150, sequenceid=130, filesize=30.3 K 2024-11-12T12:25:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/289a78e6d940462fa6b12a8509dd164c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/289a78e6d940462fa6b12a8509dd164c 2024-11-12T12:25:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,670 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/289a78e6d940462fa6b12a8509dd164c, entries=150, sequenceid=130, filesize=11.8 K 2024-11-12T12:25:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/d38d7535289b4f26b11d448b723ad694 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d38d7535289b4f26b11d448b723ad694 2024-11-12T12:25:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d38d7535289b4f26b11d448b723ad694, entries=150, sequenceid=130, filesize=11.8 K 2024-11-12T12:25:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,675 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 880bccb4d27e7b9d644e6b109920c175 in 509ms, sequenceid=130, compaction requested=false 2024-11-12T12:25:54,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:54,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:54,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-12T12:25:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-12T12:25:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-12T12:25:54,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2730 sec 2024-11-12T12:25:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.2780 sec 2024-11-12T12:25:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:55,118 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:55,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:25:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c3a89be2a80a4ff7aa8dc4b1e46e1c40_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_1/A:col10/1731414355116/Put/seqid=0 2024-11-12T12:25:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742194_1370 (size=19774) 2024-11-12T12:25:55,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414415235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414415236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414415237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414415238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414415341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414415351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414415351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414415352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55778 deadline: 1731414415371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,374 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T12:25:55,551 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:55,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-12T12:25:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T12:25:55,554 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:55,554 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:55,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:55,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414415561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414415571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414415571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414415571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,598 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:55,601 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c3a89be2a80a4ff7aa8dc4b1e46e1c40_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c3a89be2a80a4ff7aa8dc4b1e46e1c40_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:55,602 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/8f444ca6964a4c728a3ce9b227039669, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:55,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/8f444ca6964a4c728a3ce9b227039669 is 175, key is test_row_1/A:col10/1731414355116/Put/seqid=0 2024-11-12T12:25:55,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742195_1371 (size=57029) 2024-11-12T12:25:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T12:25:55,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T12:25:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:55,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:55,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T12:25:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:55,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T12:25:55,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414415871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414415891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414415892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414415892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:56,007 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/8f444ca6964a4c728a3ce9b227039669 2024-11-12T12:25:56,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/3ed4e44555e74344a8b5a39aa1bf292e is 50, key is test_row_1/B:col10/1731414355116/Put/seqid=0 2024-11-12T12:25:56,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T12:25:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742196_1372 (size=9757) 2024-11-12T12:25:56,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T12:25:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T12:25:56,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T12:25:56,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:56,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414416391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:56,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414416401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:56,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:56,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:56,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:56,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414416411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:56,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414416411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:56,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/3ed4e44555e74344a8b5a39aa1bf292e 2024-11-12T12:25:56,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/8a338339e8cb4d36ac6d05ab8b4cd02d is 50, key is test_row_1/C:col10/1731414355116/Put/seqid=0 2024-11-12T12:25:56,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742197_1373 (size=9757) 2024-11-12T12:25:56,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T12:25:56,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:56,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T12:25:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T12:25:56,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T12:25:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:56,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/8a338339e8cb4d36ac6d05ab8b4cd02d 2024-11-12T12:25:56,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/8f444ca6964a4c728a3ce9b227039669 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/8f444ca6964a4c728a3ce9b227039669 2024-11-12T12:25:56,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/8f444ca6964a4c728a3ce9b227039669, entries=300, sequenceid=144, filesize=55.7 K 2024-11-12T12:25:56,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/3ed4e44555e74344a8b5a39aa1bf292e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3ed4e44555e74344a8b5a39aa1bf292e 2024-11-12T12:25:56,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3ed4e44555e74344a8b5a39aa1bf292e, entries=100, sequenceid=144, filesize=9.5 K 2024-11-12T12:25:56,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/8a338339e8cb4d36ac6d05ab8b4cd02d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/8a338339e8cb4d36ac6d05ab8b4cd02d 2024-11-12T12:25:56,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/8a338339e8cb4d36ac6d05ab8b4cd02d, entries=100, sequenceid=144, filesize=9.5 K 2024-11-12T12:25:56,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 880bccb4d27e7b9d644e6b109920c175 in 1731ms, sequenceid=144, compaction requested=true 2024-11-12T12:25:56,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:56,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:56,849 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:56,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:56,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:56,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:56,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:56,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T12:25:56,851 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:56,852 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:56,852 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:56,852 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/B is initiating minor compaction (all files) 2024-11-12T12:25:56,852 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/A is initiating minor compaction (all files) 2024-11-12T12:25:56,852 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/B in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,852 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/A in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,852 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/bf707c34387746ce8abea46d7fb3b23f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/289a78e6d940462fa6b12a8509dd164c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3ed4e44555e74344a8b5a39aa1bf292e] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=33.3 K 2024-11-12T12:25:56,852 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/64341b3ce6f54ebb8bce960c165ecbbc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2fa8fddc847f42e4b9b2cd3349559768, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/8f444ca6964a4c728a3ce9b227039669] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=116.4 K 2024-11-12T12:25:56,852 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,852 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/64341b3ce6f54ebb8bce960c165ecbbc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2fa8fddc847f42e4b9b2cd3349559768, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/8f444ca6964a4c728a3ce9b227039669] 2024-11-12T12:25:56,852 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting bf707c34387746ce8abea46d7fb3b23f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731414351603 2024-11-12T12:25:56,853 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 289a78e6d940462fa6b12a8509dd164c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731414352788 2024-11-12T12:25:56,853 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64341b3ce6f54ebb8bce960c165ecbbc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731414351603 2024-11-12T12:25:56,855 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ed4e44555e74344a8b5a39aa1bf292e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731414355114 2024-11-12T12:25:56,855 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa8fddc847f42e4b9b2cd3349559768, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731414352788 2024-11-12T12:25:56,855 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f444ca6964a4c728a3ce9b227039669, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731414355068 2024-11-12T12:25:56,864 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:56,866 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#B#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:56,866 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/d806af049e1f4fb98424f20c27198bda is 50, key is test_row_0/B:col10/1731414352788/Put/seqid=0 2024-11-12T12:25:56,871 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411122b8d01afba62493eaca541c1ce7597bd_880bccb4d27e7b9d644e6b109920c175 store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:56,873 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411122b8d01afba62493eaca541c1ce7597bd_880bccb4d27e7b9d644e6b109920c175, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:56,873 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122b8d01afba62493eaca541c1ce7597bd_880bccb4d27e7b9d644e6b109920c175 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:56,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742198_1374 (size=12443) 2024-11-12T12:25:56,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742199_1375 (size=4469) 2024-11-12T12:25:56,880 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#A#compaction#316 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:56,881 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/371a2c41ee3142798a0e9c0cb8706344 is 175, key is test_row_0/A:col10/1731414352788/Put/seqid=0 2024-11-12T12:25:56,883 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/d806af049e1f4fb98424f20c27198bda as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d806af049e1f4fb98424f20c27198bda 2024-11-12T12:25:56,890 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/B of 880bccb4d27e7b9d644e6b109920c175 into d806af049e1f4fb98424f20c27198bda(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:56,890 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:56,890 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/B, priority=13, startTime=1731414356849; duration=0sec 2024-11-12T12:25:56,890 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:56,890 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:B 2024-11-12T12:25:56,890 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:56,891 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:56,891 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/C is initiating minor compaction (all files) 2024-11-12T12:25:56,891 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/C in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,891 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/52d606b6f652495288d91963a5e7179e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d38d7535289b4f26b11d448b723ad694, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/8a338339e8cb4d36ac6d05ab8b4cd02d] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=33.3 K 2024-11-12T12:25:56,891 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d606b6f652495288d91963a5e7179e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731414351603 2024-11-12T12:25:56,892 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d38d7535289b4f26b11d448b723ad694, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731414352788 2024-11-12T12:25:56,892 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a338339e8cb4d36ac6d05ab8b4cd02d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731414355114 2024-11-12T12:25:56,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742200_1376 (size=31504) 2024-11-12T12:25:56,902 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:56,903 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/2ac9254da4dc4b4b814d868da374bc07 is 50, key is test_row_0/C:col10/1731414352788/Put/seqid=0 2024-11-12T12:25:56,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742201_1377 (size=12443) 2024-11-12T12:25:56,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T12:25:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:56,930 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-12T12:25:56,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120429f1642c4b44bc877105b4f8118d4a_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414355234/Put/seqid=0 2024-11-12T12:25:56,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742202_1378 (size=12304) 2024-11-12T12:25:56,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:56,950 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120429f1642c4b44bc877105b4f8118d4a_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120429f1642c4b44bc877105b4f8118d4a_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:56,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/052f38086f154fd8bf79efcfc92a5a0c, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:56,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/052f38086f154fd8bf79efcfc92a5a0c is 175, key is test_row_0/A:col10/1731414355234/Put/seqid=0 2024-11-12T12:25:56,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742203_1379 (size=31105) 2024-11-12T12:25:57,297 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/371a2c41ee3142798a0e9c0cb8706344 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/371a2c41ee3142798a0e9c0cb8706344 2024-11-12T12:25:57,302 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/A of 880bccb4d27e7b9d644e6b109920c175 into 371a2c41ee3142798a0e9c0cb8706344(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:57,302 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:57,302 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/A, priority=13, startTime=1731414356849; duration=0sec 2024-11-12T12:25:57,302 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:57,302 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:A 2024-11-12T12:25:57,316 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/2ac9254da4dc4b4b814d868da374bc07 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/2ac9254da4dc4b4b814d868da374bc07 2024-11-12T12:25:57,321 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/C of 880bccb4d27e7b9d644e6b109920c175 into 2ac9254da4dc4b4b814d868da374bc07(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:57,321 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:57,321 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/C, priority=13, startTime=1731414356850; duration=0sec 2024-11-12T12:25:57,322 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:57,322 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:C 2024-11-12T12:25:57,357 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/052f38086f154fd8bf79efcfc92a5a0c 2024-11-12T12:25:57,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/a77d53773f8749389237d1f8f811b462 is 50, key is test_row_0/B:col10/1731414355234/Put/seqid=0 2024-11-12T12:25:57,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742204_1380 (size=12151) 2024-11-12T12:25:57,368 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/a77d53773f8749389237d1f8f811b462 2024-11-12T12:25:57,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/6ef804326dd2479186706ee5b555192a is 50, key is test_row_0/C:col10/1731414355234/Put/seqid=0 2024-11-12T12:25:57,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742205_1381 (size=12151) 2024-11-12T12:25:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:57,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:57,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414417427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414417431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,438 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:25:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414417431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414417438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414417541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414417551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414417551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T12:25:57,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414417751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414417761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:57,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414417761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:57,784 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/6ef804326dd2479186706ee5b555192a 2024-11-12T12:25:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/052f38086f154fd8bf79efcfc92a5a0c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/052f38086f154fd8bf79efcfc92a5a0c 2024-11-12T12:25:57,791 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/052f38086f154fd8bf79efcfc92a5a0c, entries=150, sequenceid=171, filesize=30.4 K 2024-11-12T12:25:57,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/a77d53773f8749389237d1f8f811b462 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/a77d53773f8749389237d1f8f811b462 2024-11-12T12:25:57,795 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/a77d53773f8749389237d1f8f811b462, entries=150, sequenceid=171, filesize=11.9 K 2024-11-12T12:25:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/6ef804326dd2479186706ee5b555192a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6ef804326dd2479186706ee5b555192a 2024-11-12T12:25:57,799 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6ef804326dd2479186706ee5b555192a, entries=150, sequenceid=171, filesize=11.9 K 2024-11-12T12:25:57,801 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 880bccb4d27e7b9d644e6b109920c175 in 870ms, sequenceid=171, compaction requested=false 2024-11-12T12:25:57,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:57,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:57,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-12T12:25:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-12T12:25:57,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-12T12:25:57,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2480 sec 2024-11-12T12:25:57,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.2520 sec 2024-11-12T12:25:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:58,076 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:58,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:25:58,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112052a68527a414fb6be07fb9202ddc6e9_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414358076/Put/seqid=0 2024-11-12T12:25:58,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742206_1382 (size=14794) 2024-11-12T12:25:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414418135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414418136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414418145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414418251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414418251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414418261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414418461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414418461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414418471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,493 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:58,496 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112052a68527a414fb6be07fb9202ddc6e9_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112052a68527a414fb6be07fb9202ddc6e9_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:58,497 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/2676d708ee2e4b4d8e704c73c926d3d3, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:58,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/2676d708ee2e4b4d8e704c73c926d3d3 is 175, key is test_row_0/A:col10/1731414358076/Put/seqid=0 2024-11-12T12:25:58,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742207_1383 (size=39749) 2024-11-12T12:25:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414418771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414418771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414418781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:58,902 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/2676d708ee2e4b4d8e704c73c926d3d3 2024-11-12T12:25:58,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/3e668e7b9d9449fb9de7b373881aa209 is 50, key is test_row_0/B:col10/1731414358076/Put/seqid=0 2024-11-12T12:25:58,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742208_1384 (size=12151) 2024-11-12T12:25:58,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/3e668e7b9d9449fb9de7b373881aa209 2024-11-12T12:25:58,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/dfea19b1ccec4f2a9b7ab36c5f6f8ffc is 50, key is test_row_0/C:col10/1731414358076/Put/seqid=0 2024-11-12T12:25:58,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742209_1385 (size=12151) 2024-11-12T12:25:59,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414419281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:59,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414419281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:59,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414419291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:59,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/dfea19b1ccec4f2a9b7ab36c5f6f8ffc 2024-11-12T12:25:59,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/2676d708ee2e4b4d8e704c73c926d3d3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2676d708ee2e4b4d8e704c73c926d3d3 2024-11-12T12:25:59,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2676d708ee2e4b4d8e704c73c926d3d3, entries=200, sequenceid=186, filesize=38.8 K 2024-11-12T12:25:59,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/3e668e7b9d9449fb9de7b373881aa209 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3e668e7b9d9449fb9de7b373881aa209 2024-11-12T12:25:59,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3e668e7b9d9449fb9de7b373881aa209, entries=150, sequenceid=186, filesize=11.9 K 2024-11-12T12:25:59,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/dfea19b1ccec4f2a9b7ab36c5f6f8ffc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/dfea19b1ccec4f2a9b7ab36c5f6f8ffc 2024-11-12T12:25:59,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/dfea19b1ccec4f2a9b7ab36c5f6f8ffc, entries=150, sequenceid=186, filesize=11.9 K 2024-11-12T12:25:59,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 880bccb4d27e7b9d644e6b109920c175 in 1282ms, sequenceid=186, compaction requested=true 2024-11-12T12:25:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:25:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:59,358 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:59,358 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:25:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:25:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:59,359 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102358 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:59,359 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:59,359 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/A is initiating minor compaction (all files) 2024-11-12T12:25:59,359 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/B is initiating minor compaction (all files) 2024-11-12T12:25:59,359 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/A in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:59,359 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/B in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:59,359 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d806af049e1f4fb98424f20c27198bda, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/a77d53773f8749389237d1f8f811b462, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3e668e7b9d9449fb9de7b373881aa209] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=35.9 K 2024-11-12T12:25:59,359 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/371a2c41ee3142798a0e9c0cb8706344, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/052f38086f154fd8bf79efcfc92a5a0c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2676d708ee2e4b4d8e704c73c926d3d3] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=100.0 K 2024-11-12T12:25:59,359 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:59,360 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/371a2c41ee3142798a0e9c0cb8706344, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/052f38086f154fd8bf79efcfc92a5a0c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2676d708ee2e4b4d8e704c73c926d3d3] 2024-11-12T12:25:59,360 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d806af049e1f4fb98424f20c27198bda, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731414352788 2024-11-12T12:25:59,360 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 371a2c41ee3142798a0e9c0cb8706344, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731414352788 2024-11-12T12:25:59,360 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a77d53773f8749389237d1f8f811b462, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731414355220 2024-11-12T12:25:59,360 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 052f38086f154fd8bf79efcfc92a5a0c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731414355220 2024-11-12T12:25:59,360 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2676d708ee2e4b4d8e704c73c926d3d3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731414357430 2024-11-12T12:25:59,360 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e668e7b9d9449fb9de7b373881aa209, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731414357430 2024-11-12T12:25:59,369 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:59,369 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/1c84710ddbac4e57acd0a2710ed74d79 is 50, key is test_row_0/B:col10/1731414358076/Put/seqid=0 2024-11-12T12:25:59,373 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:59,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742210_1386 (size=12595) 2024-11-12T12:25:59,385 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411120deef554f4704ecbab796b94269a5f71_880bccb4d27e7b9d644e6b109920c175 store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:59,386 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411120deef554f4704ecbab796b94269a5f71_880bccb4d27e7b9d644e6b109920c175, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:59,386 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120deef554f4704ecbab796b94269a5f71_880bccb4d27e7b9d644e6b109920c175 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:59,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742211_1387 (size=4469) 2024-11-12T12:25:59,406 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#A#compaction#325 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:59,407 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/029ea7891ca74eb19d447443f6b121b1 is 175, key is test_row_0/A:col10/1731414358076/Put/seqid=0 2024-11-12T12:25:59,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742212_1388 (size=31549) 2024-11-12T12:25:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:59,454 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:25:59,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-12T12:25:59,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123b70e710b0734ac4a228d1f0ed89fb49_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414358143/Put/seqid=0 2024-11-12T12:25:59,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742213_1389 (size=14794) 2024-11-12T12:25:59,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:25:59,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123b70e710b0734ac4a228d1f0ed89fb49_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123b70e710b0734ac4a228d1f0ed89fb49_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:25:59,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/0637212efc2d4cc891d74477eb035e31, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:25:59,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/0637212efc2d4cc891d74477eb035e31 is 175, key is test_row_0/A:col10/1731414358143/Put/seqid=0 2024-11-12T12:25:59,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742214_1390 (size=39749) 2024-11-12T12:25:59,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414419556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:59,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:59,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414419671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T12:25:59,711 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:25:59,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:25:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-12T12:25:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T12:25:59,713 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:25:59,713 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:25:59,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:25:59,785 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/1c84710ddbac4e57acd0a2710ed74d79 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/1c84710ddbac4e57acd0a2710ed74d79 2024-11-12T12:25:59,789 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/B of 880bccb4d27e7b9d644e6b109920c175 into 1c84710ddbac4e57acd0a2710ed74d79(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:59,789 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:59,789 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/B, priority=13, startTime=1731414359358; duration=0sec 2024-11-12T12:25:59,790 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:25:59,790 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:B 2024-11-12T12:25:59,790 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:25:59,791 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:25:59,791 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/C is initiating minor compaction (all files) 2024-11-12T12:25:59,791 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/C in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:59,791 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/2ac9254da4dc4b4b814d868da374bc07, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6ef804326dd2479186706ee5b555192a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/dfea19b1ccec4f2a9b7ab36c5f6f8ffc] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=35.9 K 2024-11-12T12:25:59,791 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac9254da4dc4b4b814d868da374bc07, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731414352788 2024-11-12T12:25:59,791 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef804326dd2479186706ee5b555192a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731414355220 2024-11-12T12:25:59,791 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting dfea19b1ccec4f2a9b7ab36c5f6f8ffc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731414357430 2024-11-12T12:25:59,814 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:25:59,815 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/eae11136be9147ccbe6170f4cf7c0ba1 is 50, key is test_row_0/C:col10/1731414358076/Put/seqid=0 2024-11-12T12:25:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T12:25:59,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742215_1391 (size=12595) 2024-11-12T12:25:59,833 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/029ea7891ca74eb19d447443f6b121b1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/029ea7891ca74eb19d447443f6b121b1 2024-11-12T12:25:59,833 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/eae11136be9147ccbe6170f4cf7c0ba1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eae11136be9147ccbe6170f4cf7c0ba1 2024-11-12T12:25:59,837 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/A of 880bccb4d27e7b9d644e6b109920c175 into 029ea7891ca74eb19d447443f6b121b1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:59,837 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/C of 880bccb4d27e7b9d644e6b109920c175 into eae11136be9147ccbe6170f4cf7c0ba1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:25:59,837 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:59,837 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:25:59,837 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/A, priority=13, startTime=1731414359358; duration=0sec 2024-11-12T12:25:59,837 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/C, priority=13, startTime=1731414359358; duration=0sec 2024-11-12T12:25:59,837 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:59,837 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:25:59,837 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:A 2024-11-12T12:25:59,837 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:C 2024-11-12T12:25:59,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T12:25:59,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:59,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:25:59,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:25:59,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:59,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:25:59,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:25:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414419881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:25:59,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/0637212efc2d4cc891d74477eb035e31 2024-11-12T12:25:59,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:25:59,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:25:59,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/98351f29b56844339c32bd6f8095f46e is 50, key is test_row_0/B:col10/1731414358143/Put/seqid=0 2024-11-12T12:25:59,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742216_1392 (size=12151) 2024-11-12T12:26:00,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T12:26:00,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:00,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T12:26:00,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T12:26:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414420201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:00,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414420291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:00,295 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:00,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414420301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:00,305 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:00,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:00,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414420311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:00,313 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:00,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/98351f29b56844339c32bd6f8095f46e 2024-11-12T12:26:00,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/779c41e92ac746bd995544961c273a56 is 50, key is test_row_0/C:col10/1731414358143/Put/seqid=0 2024-11-12T12:26:00,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T12:26:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742217_1393 (size=12151) 2024-11-12T12:26:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T12:26:00,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T12:26:00,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:00,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T12:26:00,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:00,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414420721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:00,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:00,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/779c41e92ac746bd995544961c273a56 2024-11-12T12:26:00,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/0637212efc2d4cc891d74477eb035e31 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/0637212efc2d4cc891d74477eb035e31 2024-11-12T12:26:00,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/0637212efc2d4cc891d74477eb035e31, entries=200, sequenceid=210, filesize=38.8 K 2024-11-12T12:26:00,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/98351f29b56844339c32bd6f8095f46e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/98351f29b56844339c32bd6f8095f46e 2024-11-12T12:26:00,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/98351f29b56844339c32bd6f8095f46e, entries=150, sequenceid=210, filesize=11.9 K 2024-11-12T12:26:00,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/779c41e92ac746bd995544961c273a56 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/779c41e92ac746bd995544961c273a56 2024-11-12T12:26:00,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/779c41e92ac746bd995544961c273a56, entries=150, sequenceid=210, filesize=11.9 K 2024-11-12T12:26:00,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 880bccb4d27e7b9d644e6b109920c175 in 1293ms, sequenceid=210, compaction requested=false 2024-11-12T12:26:00,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:00,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T12:26:00,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:00,781 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-12T12:26:00,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128e53f6e970ad478fae93515042fbe114_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414359553/Put/seqid=0 2024-11-12T12:26:00,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742218_1394 (size=12304) 2024-11-12T12:26:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T12:26:01,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:01,197 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128e53f6e970ad478fae93515042fbe114_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128e53f6e970ad478fae93515042fbe114_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:01,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/81ab2b5ef5654509b32a1cfdf4e35fd1, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:01,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/81ab2b5ef5654509b32a1cfdf4e35fd1 is 175, key is test_row_0/A:col10/1731414359553/Put/seqid=0 2024-11-12T12:26:01,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742219_1395 (size=31105) 2024-11-12T12:26:01,609 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/81ab2b5ef5654509b32a1cfdf4e35fd1 2024-11-12T12:26:01,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/edbdf71e7c29445cbddc79e4f3a0eef9 is 50, key is test_row_0/B:col10/1731414359553/Put/seqid=0 2024-11-12T12:26:01,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742220_1396 (size=12151) 2024-11-12T12:26:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:01,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T12:26:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414421892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:02,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414422001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:02,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:02,025 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/edbdf71e7c29445cbddc79e4f3a0eef9 2024-11-12T12:26:02,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/3c45a62568e746e0971d776f3e018adc is 50, key is test_row_0/C:col10/1731414359553/Put/seqid=0 2024-11-12T12:26:02,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742221_1397 (size=12151) 2024-11-12T12:26:02,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414422211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:02,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414422321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:02,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414422322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:02,326 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,326 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:02,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414422331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:02,335 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:02,443 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/3c45a62568e746e0971d776f3e018adc 2024-11-12T12:26:02,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/81ab2b5ef5654509b32a1cfdf4e35fd1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/81ab2b5ef5654509b32a1cfdf4e35fd1 2024-11-12T12:26:02,452 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/81ab2b5ef5654509b32a1cfdf4e35fd1, entries=150, sequenceid=225, filesize=30.4 K 2024-11-12T12:26:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/edbdf71e7c29445cbddc79e4f3a0eef9 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/edbdf71e7c29445cbddc79e4f3a0eef9 2024-11-12T12:26:02,456 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/edbdf71e7c29445cbddc79e4f3a0eef9, entries=150, sequenceid=225, filesize=11.9 K 2024-11-12T12:26:02,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/3c45a62568e746e0971d776f3e018adc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3c45a62568e746e0971d776f3e018adc 2024-11-12T12:26:02,461 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3c45a62568e746e0971d776f3e018adc, entries=150, sequenceid=225, filesize=11.9 K 2024-11-12T12:26:02,462 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 880bccb4d27e7b9d644e6b109920c175 in 1681ms, sequenceid=225, compaction requested=true 2024-11-12T12:26:02,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:02,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:02,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-12T12:26:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-12T12:26:02,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-12T12:26:02,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7500 sec 2024-11-12T12:26:02,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.7530 sec 2024-11-12T12:26:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:02,527 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:02,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:26:02,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129f3e2f7d7ab84c589c1c272a1ac2002b_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414361890/Put/seqid=0 2024-11-12T12:26:02,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742222_1398 (size=14794) 2024-11-12T12:26:02,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414422583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:02,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414422691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:02,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414422901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:02,952 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:02,955 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129f3e2f7d7ab84c589c1c272a1ac2002b_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129f3e2f7d7ab84c589c1c272a1ac2002b_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:02,956 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/a8b1a5dd3b4248a18328ae2053b093b7, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:02,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/a8b1a5dd3b4248a18328ae2053b093b7 is 175, key is test_row_0/A:col10/1731414361890/Put/seqid=0 2024-11-12T12:26:02,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742223_1399 (size=39749) 2024-11-12T12:26:03,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414423221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:03,366 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/a8b1a5dd3b4248a18328ae2053b093b7 2024-11-12T12:26:03,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/0981994db1e6493db87953ed9a7c898a is 50, key is test_row_0/B:col10/1731414361890/Put/seqid=0 2024-11-12T12:26:03,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742224_1400 (size=12151) 2024-11-12T12:26:03,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414423741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:03,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/0981994db1e6493db87953ed9a7c898a 2024-11-12T12:26:03,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/ba5392221eb149739187565d3087ded7 is 50, key is test_row_0/C:col10/1731414361890/Put/seqid=0 2024-11-12T12:26:03,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742225_1401 (size=12151) 2024-11-12T12:26:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T12:26:03,882 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:03,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-12T12:26:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T12:26:03,883 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:03,884 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:03,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T12:26:04,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T12:26:04,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:04,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:04,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:04,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:04,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:04,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T12:26:04,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:04,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:04,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:04,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:04,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:04,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/ba5392221eb149739187565d3087ded7 2024-11-12T12:26:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T12:26:04,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/a8b1a5dd3b4248a18328ae2053b093b7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/a8b1a5dd3b4248a18328ae2053b093b7 2024-11-12T12:26:04,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/a8b1a5dd3b4248a18328ae2053b093b7, entries=200, sequenceid=250, filesize=38.8 K 2024-11-12T12:26:04,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/0981994db1e6493db87953ed9a7c898a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/0981994db1e6493db87953ed9a7c898a 2024-11-12T12:26:04,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/0981994db1e6493db87953ed9a7c898a, entries=150, sequenceid=250, filesize=11.9 K 2024-11-12T12:26:04,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/ba5392221eb149739187565d3087ded7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ba5392221eb149739187565d3087ded7 2024-11-12T12:26:04,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ba5392221eb149739187565d3087ded7, entries=150, sequenceid=250, filesize=11.9 K 2024-11-12T12:26:04,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 880bccb4d27e7b9d644e6b109920c175 in 1691ms, sequenceid=250, compaction requested=true 2024-11-12T12:26:04,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:04,219 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:04,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:04,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:04,219 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:04,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:04,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:04,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:04,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:04,222 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:04,222 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:04,222 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/A is initiating minor compaction (all files) 2024-11-12T12:26:04,222 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/B is initiating minor compaction (all files) 2024-11-12T12:26:04,222 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/B in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:04,222 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/A in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:04,222 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/029ea7891ca74eb19d447443f6b121b1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/0637212efc2d4cc891d74477eb035e31, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/81ab2b5ef5654509b32a1cfdf4e35fd1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/a8b1a5dd3b4248a18328ae2053b093b7] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=138.8 K 2024-11-12T12:26:04,222 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/1c84710ddbac4e57acd0a2710ed74d79, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/98351f29b56844339c32bd6f8095f46e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/edbdf71e7c29445cbddc79e4f3a0eef9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/0981994db1e6493db87953ed9a7c898a] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=47.9 K 2024-11-12T12:26:04,222 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:04,222 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/029ea7891ca74eb19d447443f6b121b1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/0637212efc2d4cc891d74477eb035e31, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/81ab2b5ef5654509b32a1cfdf4e35fd1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/a8b1a5dd3b4248a18328ae2053b093b7] 2024-11-12T12:26:04,222 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c84710ddbac4e57acd0a2710ed74d79, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731414357430 2024-11-12T12:26:04,223 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 98351f29b56844339c32bd6f8095f46e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731414358118 2024-11-12T12:26:04,223 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 029ea7891ca74eb19d447443f6b121b1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731414357430 2024-11-12T12:26:04,223 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting edbdf71e7c29445cbddc79e4f3a0eef9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731414359526 2024-11-12T12:26:04,223 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0637212efc2d4cc891d74477eb035e31, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731414358118 2024-11-12T12:26:04,224 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 0981994db1e6493db87953ed9a7c898a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731414361855 2024-11-12T12:26:04,224 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ab2b5ef5654509b32a1cfdf4e35fd1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731414359526 2024-11-12T12:26:04,224 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8b1a5dd3b4248a18328ae2053b093b7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731414361855 2024-11-12T12:26:04,233 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:04,238 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#B#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:04,239 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/f9a2ef9e61a244ba97320e34e5551687 is 50, key is test_row_0/B:col10/1731414361890/Put/seqid=0 2024-11-12T12:26:04,245 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112d015082144d64bd18c00a8be13b54e90_880bccb4d27e7b9d644e6b109920c175 store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:04,248 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112d015082144d64bd18c00a8be13b54e90_880bccb4d27e7b9d644e6b109920c175, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:04,248 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d015082144d64bd18c00a8be13b54e90_880bccb4d27e7b9d644e6b109920c175 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:04,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742226_1402 (size=12731) 2024-11-12T12:26:04,266 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/f9a2ef9e61a244ba97320e34e5551687 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f9a2ef9e61a244ba97320e34e5551687 2024-11-12T12:26:04,272 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/B of 880bccb4d27e7b9d644e6b109920c175 into f9a2ef9e61a244ba97320e34e5551687(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:04,272 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:04,272 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/B, priority=12, startTime=1731414364219; duration=0sec 2024-11-12T12:26:04,272 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:04,272 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:B 2024-11-12T12:26:04,272 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:04,275 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:04,275 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/C is initiating minor compaction (all files) 2024-11-12T12:26:04,275 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/C in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:04,276 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eae11136be9147ccbe6170f4cf7c0ba1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/779c41e92ac746bd995544961c273a56, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3c45a62568e746e0971d776f3e018adc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ba5392221eb149739187565d3087ded7] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=47.9 K 2024-11-12T12:26:04,276 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting eae11136be9147ccbe6170f4cf7c0ba1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731414357430 2024-11-12T12:26:04,276 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 779c41e92ac746bd995544961c273a56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731414358118 2024-11-12T12:26:04,276 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c45a62568e746e0971d776f3e018adc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731414359526 2024-11-12T12:26:04,277 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ba5392221eb149739187565d3087ded7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731414361855 2024-11-12T12:26:04,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742227_1403 (size=4469) 2024-11-12T12:26:04,288 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#A#compaction#336 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:04,288 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/7adeeb23cc84472da5fb6c229b6f28cf is 175, key is test_row_0/A:col10/1731414361890/Put/seqid=0 2024-11-12T12:26:04,289 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:04,291 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/a6b28a1793b3456a953faf6cd8b38b36 is 50, key is test_row_0/C:col10/1731414361890/Put/seqid=0 2024-11-12T12:26:04,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742228_1404 (size=31685) 2024-11-12T12:26:04,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742229_1405 (size=12731) 2024-11-12T12:26:04,308 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/a6b28a1793b3456a953faf6cd8b38b36 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a6b28a1793b3456a953faf6cd8b38b36 2024-11-12T12:26:04,312 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/C of 880bccb4d27e7b9d644e6b109920c175 into a6b28a1793b3456a953faf6cd8b38b36(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:04,312 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:04,312 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/C, priority=12, startTime=1731414364220; duration=0sec 2024-11-12T12:26:04,312 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:04,312 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:C 2024-11-12T12:26:04,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T12:26:04,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:04,342 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:26:04,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bab098699f4d4715a68e82675558d144_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414362573/Put/seqid=0 2024-11-12T12:26:04,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742230_1406 (size=12454) 2024-11-12T12:26:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T12:26:04,700 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/7adeeb23cc84472da5fb6c229b6f28cf as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/7adeeb23cc84472da5fb6c229b6f28cf 2024-11-12T12:26:04,704 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/A of 880bccb4d27e7b9d644e6b109920c175 into 7adeeb23cc84472da5fb6c229b6f28cf(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:04,704 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:04,704 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/A, priority=12, startTime=1731414364219; duration=0sec 2024-11-12T12:26:04,705 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:04,705 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:A 2024-11-12T12:26:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:04,762 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bab098699f4d4715a68e82675558d144_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bab098699f4d4715a68e82675558d144_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/bfe5705bd064471abab7aa21e576fa79, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/bfe5705bd064471abab7aa21e576fa79 is 175, key is test_row_0/A:col10/1731414362573/Put/seqid=0 2024-11-12T12:26:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:04,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:04,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742231_1407 (size=31255) 2024-11-12T12:26:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414424889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:05,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414425001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T12:26:05,170 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/bfe5705bd064471abab7aa21e576fa79 2024-11-12T12:26:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/f24a7a16c4f2495faa78b1fd4fdd572b is 50, key is test_row_0/B:col10/1731414362573/Put/seqid=0 2024-11-12T12:26:05,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742232_1408 (size=12301) 2024-11-12T12:26:05,189 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/f24a7a16c4f2495faa78b1fd4fdd572b 2024-11-12T12:26:05,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/099195b25224442da678db9ce297ffb8 is 50, key is test_row_0/C:col10/1731414362573/Put/seqid=0 2024-11-12T12:26:05,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742233_1409 (size=12301) 2024-11-12T12:26:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414425211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:05,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55778 deadline: 1731414425461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:05,463 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18304 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:05,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414425531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:05,599 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/099195b25224442da678db9ce297ffb8 2024-11-12T12:26:05,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/bfe5705bd064471abab7aa21e576fa79 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/bfe5705bd064471abab7aa21e576fa79 2024-11-12T12:26:05,606 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/bfe5705bd064471abab7aa21e576fa79, entries=150, sequenceid=263, filesize=30.5 K 2024-11-12T12:26:05,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/f24a7a16c4f2495faa78b1fd4fdd572b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f24a7a16c4f2495faa78b1fd4fdd572b 2024-11-12T12:26:05,611 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f24a7a16c4f2495faa78b1fd4fdd572b, entries=150, sequenceid=263, filesize=12.0 K 2024-11-12T12:26:05,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/099195b25224442da678db9ce297ffb8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/099195b25224442da678db9ce297ffb8 2024-11-12T12:26:05,615 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/099195b25224442da678db9ce297ffb8, entries=150, sequenceid=263, filesize=12.0 K 2024-11-12T12:26:05,616 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 880bccb4d27e7b9d644e6b109920c175 in 1275ms, sequenceid=263, compaction requested=false 2024-11-12T12:26:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-12T12:26:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-12T12:26:05,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-12T12:26:05,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7330 sec 2024-11-12T12:26:05,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.7380 sec 2024-11-12T12:26:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T12:26:06,031 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:06,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-12T12:26:06,033 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T12:26:06,033 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:06,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:06,047 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:06,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-12T12:26:06,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123e8bfdb01e034caabd7a14d94ec9100e_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414364858/Put/seqid=0 2024-11-12T12:26:06,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742234_1410 (size=14994) 2024-11-12T12:26:06,063 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:06,070 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123e8bfdb01e034caabd7a14d94ec9100e_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123e8bfdb01e034caabd7a14d94ec9100e_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:06,071 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/5b9a53e7b2624b0b9c9b990a51edbe0d, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:06,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/5b9a53e7b2624b0b9c9b990a51edbe0d is 175, key is test_row_0/A:col10/1731414364858/Put/seqid=0 2024-11-12T12:26:06,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742235_1411 (size=39949) 2024-11-12T12:26:06,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414426100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:06,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T12:26:06,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T12:26:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:06,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414426211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:06,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T12:26:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55774 deadline: 1731414426341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:06,348 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T12:26:06,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55758 deadline: 1731414426351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:06,361 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:06,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55786 deadline: 1731414426371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:06,375 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in 880bccb4d27e7b9d644e6b109920c175 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414426421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:06,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/5b9a53e7b2624b0b9c9b990a51edbe0d 2024-11-12T12:26:06,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/2c4c2a0562b94610b107f1eb11568fef is 50, key is test_row_0/B:col10/1731414364858/Put/seqid=0 2024-11-12T12:26:06,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T12:26:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:06,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742236_1412 (size=12301) 2024-11-12T12:26:06,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/2c4c2a0562b94610b107f1eb11568fef 2024-11-12T12:26:06,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/a7320f1acaa34b73bb06c29836237cb8 is 50, key is test_row_0/C:col10/1731414364858/Put/seqid=0 2024-11-12T12:26:06,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742237_1413 (size=12301) 2024-11-12T12:26:06,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/a7320f1acaa34b73bb06c29836237cb8 2024-11-12T12:26:06,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/5b9a53e7b2624b0b9c9b990a51edbe0d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/5b9a53e7b2624b0b9c9b990a51edbe0d 2024-11-12T12:26:06,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/5b9a53e7b2624b0b9c9b990a51edbe0d, entries=200, sequenceid=290, filesize=39.0 K 2024-11-12T12:26:06,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/2c4c2a0562b94610b107f1eb11568fef as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2c4c2a0562b94610b107f1eb11568fef 2024-11-12T12:26:06,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2c4c2a0562b94610b107f1eb11568fef, entries=150, sequenceid=290, filesize=12.0 K 2024-11-12T12:26:06,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/a7320f1acaa34b73bb06c29836237cb8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7320f1acaa34b73bb06c29836237cb8 2024-11-12T12:26:06,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7320f1acaa34b73bb06c29836237cb8, entries=150, sequenceid=290, filesize=12.0 K 2024-11-12T12:26:06,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 880bccb4d27e7b9d644e6b109920c175 in 495ms, sequenceid=290, compaction requested=true 2024-11-12T12:26:06,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:06,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:06,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:06,542 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:06,542 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:06,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:06,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:06,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:06,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:06,543 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:06,543 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/A is initiating minor compaction (all files) 2024-11-12T12:26:06,543 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/A in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,543 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/7adeeb23cc84472da5fb6c229b6f28cf, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/bfe5705bd064471abab7aa21e576fa79, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/5b9a53e7b2624b0b9c9b990a51edbe0d] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=100.5 K 2024-11-12T12:26:06,543 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,543 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/7adeeb23cc84472da5fb6c229b6f28cf, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/bfe5705bd064471abab7aa21e576fa79, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/5b9a53e7b2624b0b9c9b990a51edbe0d] 2024-11-12T12:26:06,544 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:06,544 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/B is initiating minor compaction (all files) 2024-11-12T12:26:06,544 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/B in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,544 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7adeeb23cc84472da5fb6c229b6f28cf, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731414361855 2024-11-12T12:26:06,544 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f9a2ef9e61a244ba97320e34e5551687, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f24a7a16c4f2495faa78b1fd4fdd572b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2c4c2a0562b94610b107f1eb11568fef] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=36.5 K 2024-11-12T12:26:06,544 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfe5705bd064471abab7aa21e576fa79, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731414362558 2024-11-12T12:26:06,544 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting f9a2ef9e61a244ba97320e34e5551687, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731414361855 2024-11-12T12:26:06,544 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b9a53e7b2624b0b9c9b990a51edbe0d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414364789 2024-11-12T12:26:06,544 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting f24a7a16c4f2495faa78b1fd4fdd572b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731414362558 2024-11-12T12:26:06,545 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c4c2a0562b94610b107f1eb11568fef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414364789 2024-11-12T12:26:06,553 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:06,553 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/c5b4381b9b3c4c21a55b5eb766f7ca28 is 50, key is test_row_0/B:col10/1731414364858/Put/seqid=0 2024-11-12T12:26:06,555 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:06,561 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112fc3ff6a429ce4b6caaf1857ba99908db_880bccb4d27e7b9d644e6b109920c175 store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:06,563 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112fc3ff6a429ce4b6caaf1857ba99908db_880bccb4d27e7b9d644e6b109920c175, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:06,563 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fc3ff6a429ce4b6caaf1857ba99908db_880bccb4d27e7b9d644e6b109920c175 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:06,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742238_1414 (size=12983) 2024-11-12T12:26:06,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742239_1415 (size=4469) 2024-11-12T12:26:06,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T12:26:06,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,644 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-12T12:26:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121873954e384946898a17694026d10839_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414366098/Put/seqid=0 2024-11-12T12:26:06,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742240_1416 (size=12454) 2024-11-12T12:26:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T12:26:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:06,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. as already flushing 2024-11-12T12:26:06,973 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/c5b4381b9b3c4c21a55b5eb766f7ca28 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c5b4381b9b3c4c21a55b5eb766f7ca28 2024-11-12T12:26:06,975 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:06,975 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/dc5c3999a6f14155afc3fdd6c330f6ae is 175, key is test_row_0/A:col10/1731414364858/Put/seqid=0 2024-11-12T12:26:06,977 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/B of 880bccb4d27e7b9d644e6b109920c175 into c5b4381b9b3c4c21a55b5eb766f7ca28(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:06,977 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:06,978 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/B, priority=13, startTime=1731414366542; duration=0sec 2024-11-12T12:26:06,978 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:06,978 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:B 2024-11-12T12:26:06,978 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:06,980 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:06,980 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/C is initiating minor compaction (all files) 2024-11-12T12:26:06,980 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/C in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:06,981 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a6b28a1793b3456a953faf6cd8b38b36, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/099195b25224442da678db9ce297ffb8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7320f1acaa34b73bb06c29836237cb8] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=36.5 K 2024-11-12T12:26:06,981 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a6b28a1793b3456a953faf6cd8b38b36, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731414361855 2024-11-12T12:26:06,982 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 099195b25224442da678db9ce297ffb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731414362558 2024-11-12T12:26:06,982 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a7320f1acaa34b73bb06c29836237cb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414364789 2024-11-12T12:26:06,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742241_1417 (size=31937) 2024-11-12T12:26:06,995 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/dc5c3999a6f14155afc3fdd6c330f6ae as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/dc5c3999a6f14155afc3fdd6c330f6ae 2024-11-12T12:26:06,999 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/A of 880bccb4d27e7b9d644e6b109920c175 into dc5c3999a6f14155afc3fdd6c330f6ae(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:06,999 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:06,999 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/A, priority=13, startTime=1731414366542; duration=0sec 2024-11-12T12:26:07,000 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:07,000 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:A 2024-11-12T12:26:07,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414426989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:07,006 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#C#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:07,007 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/5b06cccaa82b45e99597416391c44d2f is 50, key is test_row_0/C:col10/1731414364858/Put/seqid=0 2024-11-12T12:26:07,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742242_1418 (size=12983) 2024-11-12T12:26:07,036 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/5b06cccaa82b45e99597416391c44d2f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/5b06cccaa82b45e99597416391c44d2f 2024-11-12T12:26:07,044 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/C of 880bccb4d27e7b9d644e6b109920c175 into 5b06cccaa82b45e99597416391c44d2f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:07,044 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:07,044 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/C, priority=13, startTime=1731414366542; duration=0sec 2024-11-12T12:26:07,044 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:07,044 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:C 2024-11-12T12:26:07,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:07,066 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121873954e384946898a17694026d10839_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121873954e384946898a17694026d10839_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:07,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/4e095bd4ede749488d8e9a54782d1ac5, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:07,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/4e095bd4ede749488d8e9a54782d1ac5 is 175, key is test_row_0/A:col10/1731414366098/Put/seqid=0 2024-11-12T12:26:07,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742243_1419 (size=31255) 2024-11-12T12:26:07,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414427111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:07,119 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-11-12T12:26:07,119 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:07,119 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,119 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:07,123 INFO [Thread-1532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1532. 2024-11-12T12:26:07,123 DEBUG [Thread-1532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:07,123 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,123 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:07,125 INFO [Thread-1534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1534. 2024-11-12T12:26:07,125 DEBUG [Thread-1534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:07,125 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,125 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:07,130 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-11-12T12:26:07,130 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:07,130 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,130 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,130 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:07,133 INFO [Thread-1530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1530. 2024-11-12T12:26:07,133 DEBUG [Thread-1530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:07,133 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,133 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:07,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T12:26:07,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414427322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:07,473 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/4e095bd4ede749488d8e9a54782d1ac5 2024-11-12T12:26:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/2a28fc087929482ab651f787ee5b9067 is 50, key is test_row_0/B:col10/1731414366098/Put/seqid=0 2024-11-12T12:26:07,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742244_1420 (size=12301) 2024-11-12T12:26:07,484 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/2a28fc087929482ab651f787ee5b9067 2024-11-12T12:26:07,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/f7ee26e233694cc2917d04d68ef11022 is 50, key is test_row_0/C:col10/1731414366098/Put/seqid=0 2024-11-12T12:26:07,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742245_1421 (size=12301) 2024-11-12T12:26:07,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55748 deadline: 1731414427631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:07,900 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/f7ee26e233694cc2917d04d68ef11022 2024-11-12T12:26:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/4e095bd4ede749488d8e9a54782d1ac5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/4e095bd4ede749488d8e9a54782d1ac5 2024-11-12T12:26:07,906 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/4e095bd4ede749488d8e9a54782d1ac5, entries=150, sequenceid=300, filesize=30.5 K 2024-11-12T12:26:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/2a28fc087929482ab651f787ee5b9067 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2a28fc087929482ab651f787ee5b9067 2024-11-12T12:26:07,910 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2a28fc087929482ab651f787ee5b9067, entries=150, sequenceid=300, filesize=12.0 K 2024-11-12T12:26:07,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/f7ee26e233694cc2917d04d68ef11022 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/f7ee26e233694cc2917d04d68ef11022 2024-11-12T12:26:07,913 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/f7ee26e233694cc2917d04d68ef11022, entries=150, sequenceid=300, filesize=12.0 K 2024-11-12T12:26:07,914 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 880bccb4d27e7b9d644e6b109920c175 in 1270ms, sequenceid=300, compaction requested=false 2024-11-12T12:26:07,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:07,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:07,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-12T12:26:07,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-12T12:26:07,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-12T12:26:07,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-12T12:26:07,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8860 sec 2024-11-12T12:26:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:08,143 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:08,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-12T12:26:08,143 INFO [Thread-1521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1521. 2024-11-12T12:26:08,143 DEBUG [Thread-1521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:08,143 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:08,143 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:08,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:08,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111214e7af7d9777443985615bfba2aeeef6_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414368142/Put/seqid=0 2024-11-12T12:26:08,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742246_1422 (size=12454) 2024-11-12T12:26:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T12:26:08,181 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:08,551 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:08,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111214e7af7d9777443985615bfba2aeeef6_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111214e7af7d9777443985615bfba2aeeef6_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:08,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/410be8630a194b5db20a00c7ac7362e4, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:08,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/410be8630a194b5db20a00c7ac7362e4 is 175, key is test_row_0/A:col10/1731414368142/Put/seqid=0 2024-11-12T12:26:08,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742247_1423 (size=31255) 2024-11-12T12:26:08,959 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/410be8630a194b5db20a00c7ac7362e4 2024-11-12T12:26:08,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/7358eb559b174af6a7bf7e61c35f7cbe is 50, key is test_row_0/B:col10/1731414368142/Put/seqid=0 2024-11-12T12:26:08,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742248_1424 (size=12301) 2024-11-12T12:26:09,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/7358eb559b174af6a7bf7e61c35f7cbe 2024-11-12T12:26:09,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/41b03e9588e24e458f85cc78c1e055a6 is 50, key is test_row_0/C:col10/1731414368142/Put/seqid=0 2024-11-12T12:26:09,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742249_1425 (size=12301) 2024-11-12T12:26:09,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/41b03e9588e24e458f85cc78c1e055a6 2024-11-12T12:26:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/410be8630a194b5db20a00c7ac7362e4 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/410be8630a194b5db20a00c7ac7362e4 2024-11-12T12:26:09,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/410be8630a194b5db20a00c7ac7362e4, entries=150, sequenceid=330, filesize=30.5 K 2024-11-12T12:26:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/7358eb559b174af6a7bf7e61c35f7cbe as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/7358eb559b174af6a7bf7e61c35f7cbe 2024-11-12T12:26:09,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/7358eb559b174af6a7bf7e61c35f7cbe, entries=150, sequenceid=330, filesize=12.0 K 2024-11-12T12:26:09,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/41b03e9588e24e458f85cc78c1e055a6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/41b03e9588e24e458f85cc78c1e055a6 2024-11-12T12:26:09,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/41b03e9588e24e458f85cc78c1e055a6, entries=150, sequenceid=330, filesize=12.0 K 2024-11-12T12:26:09,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=0 B/0 for 880bccb4d27e7b9d644e6b109920c175 in 1655ms, sequenceid=330, compaction requested=true 2024-11-12T12:26:09,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:09,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:09,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:09,798 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:09,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:09,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:09,798 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:09,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 880bccb4d27e7b9d644e6b109920c175:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:09,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:09,799 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:09,799 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:09,799 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/A is initiating minor compaction (all files) 2024-11-12T12:26:09,799 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/B is initiating minor compaction (all files) 2024-11-12T12:26:09,799 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/B in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:09,799 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/A in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:09,799 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/dc5c3999a6f14155afc3fdd6c330f6ae, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/4e095bd4ede749488d8e9a54782d1ac5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/410be8630a194b5db20a00c7ac7362e4] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=92.2 K 2024-11-12T12:26:09,799 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c5b4381b9b3c4c21a55b5eb766f7ca28, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2a28fc087929482ab651f787ee5b9067, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/7358eb559b174af6a7bf7e61c35f7cbe] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=36.7 K 2024-11-12T12:26:09,799 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:09,799 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/dc5c3999a6f14155afc3fdd6c330f6ae, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/4e095bd4ede749488d8e9a54782d1ac5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/410be8630a194b5db20a00c7ac7362e4] 2024-11-12T12:26:09,799 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting c5b4381b9b3c4c21a55b5eb766f7ca28, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414364789 2024-11-12T12:26:09,799 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc5c3999a6f14155afc3fdd6c330f6ae, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414364789 2024-11-12T12:26:09,800 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a28fc087929482ab651f787ee5b9067, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731414366061 2024-11-12T12:26:09,800 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e095bd4ede749488d8e9a54782d1ac5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731414366061 2024-11-12T12:26:09,800 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 7358eb559b174af6a7bf7e61c35f7cbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731414366916 2024-11-12T12:26:09,800 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 410be8630a194b5db20a00c7ac7362e4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731414366916 2024-11-12T12:26:09,805 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:09,806 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:09,806 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112092aa6e9a5ae41a5bc3150c710282068_880bccb4d27e7b9d644e6b109920c175 store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:09,806 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/3842d2b48cf5426dae7604642a3f6bbc is 50, key is test_row_0/B:col10/1731414368142/Put/seqid=0 2024-11-12T12:26:09,809 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112092aa6e9a5ae41a5bc3150c710282068_880bccb4d27e7b9d644e6b109920c175, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:09,809 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112092aa6e9a5ae41a5bc3150c710282068_880bccb4d27e7b9d644e6b109920c175 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:09,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742250_1426 (size=13085) 2024-11-12T12:26:09,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742251_1427 (size=4469) 2024-11-12T12:26:10,214 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/3842d2b48cf5426dae7604642a3f6bbc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3842d2b48cf5426dae7604642a3f6bbc 2024-11-12T12:26:10,217 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/B of 880bccb4d27e7b9d644e6b109920c175 into 3842d2b48cf5426dae7604642a3f6bbc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:10,218 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:10,218 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/B, priority=13, startTime=1731414369798; duration=0sec 2024-11-12T12:26:10,218 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:10,218 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:B 2024-11-12T12:26:10,218 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:10,218 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:10,218 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): 880bccb4d27e7b9d644e6b109920c175/C is initiating minor compaction (all files) 2024-11-12T12:26:10,218 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 880bccb4d27e7b9d644e6b109920c175/C in TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:10,219 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/5b06cccaa82b45e99597416391c44d2f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/f7ee26e233694cc2917d04d68ef11022, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/41b03e9588e24e458f85cc78c1e055a6] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp, totalSize=36.7 K 2024-11-12T12:26:10,219 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b06cccaa82b45e99597416391c44d2f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731414364789 2024-11-12T12:26:10,219 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ee26e233694cc2917d04d68ef11022, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731414366061 2024-11-12T12:26:10,219 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b03e9588e24e458f85cc78c1e055a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731414366916 2024-11-12T12:26:10,220 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#A#compaction#355 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:10,220 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/3f5b1cc132954257ba3e8402838f22ac is 175, key is test_row_0/A:col10/1731414368142/Put/seqid=0 2024-11-12T12:26:10,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742252_1428 (size=32039) 2024-11-12T12:26:10,226 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 880bccb4d27e7b9d644e6b109920c175#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:10,227 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/f31ba5bd6d0d4a85b824ba9a9fa58113 is 50, key is test_row_0/C:col10/1731414368142/Put/seqid=0 2024-11-12T12:26:10,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742253_1429 (size=13085) 2024-11-12T12:26:10,627 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/3f5b1cc132954257ba3e8402838f22ac as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3f5b1cc132954257ba3e8402838f22ac 2024-11-12T12:26:10,631 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/A of 880bccb4d27e7b9d644e6b109920c175 into 3f5b1cc132954257ba3e8402838f22ac(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:10,631 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:10,631 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/A, priority=13, startTime=1731414369798; duration=0sec 2024-11-12T12:26:10,631 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:10,631 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:A 2024-11-12T12:26:10,633 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/f31ba5bd6d0d4a85b824ba9a9fa58113 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/f31ba5bd6d0d4a85b824ba9a9fa58113 2024-11-12T12:26:10,636 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 880bccb4d27e7b9d644e6b109920c175/C of 880bccb4d27e7b9d644e6b109920c175 into f31ba5bd6d0d4a85b824ba9a9fa58113(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:10,636 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 880bccb4d27e7b9d644e6b109920c175: 2024-11-12T12:26:10,636 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., storeName=880bccb4d27e7b9d644e6b109920c175/C, priority=13, startTime=1731414369798; duration=0sec 2024-11-12T12:26:10,636 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:10,636 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 880bccb4d27e7b9d644e6b109920c175:C 2024-11-12T12:26:15,472 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-11-12T12:26:15,472 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:15,472 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:15,472 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:15,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:16,046 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T12:26:16,402 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-11-12T12:26:16,402 INFO [Thread-1519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1519. 2024-11-12T12:26:16,402 DEBUG [Thread-1519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:16,402 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:16,402 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:16,402 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:16,403 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:16,403 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:16,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:16,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:16,433 INFO [Thread-1523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1523. 2024-11-12T12:26:16,433 DEBUG [Thread-1523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:16,433 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:16,433 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:16,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1337 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4006 rows 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1345 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3986 rows 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1332 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3996 rows 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1345 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3983 rows 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1349 2024-11-12T12:26:16,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4009 rows 2024-11-12T12:26:16,435 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T12:26:16,435 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T12:26:16,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:16,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:16,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:16,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T12:26:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T12:26:16,439 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414376438"}]},"ts":"1731414376438"} 2024-11-12T12:26:16,440 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T12:26:16,440 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T12:26:16,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T12:26:16,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, UNASSIGN}] 2024-11-12T12:26:16,442 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, UNASSIGN 2024-11-12T12:26:16,442 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=880bccb4d27e7b9d644e6b109920c175, regionState=CLOSING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:16,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, UNASSIGN because future has completed 2024-11-12T12:26:16,443 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T12:26:16,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:26:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T12:26:16,595 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:16,596 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T12:26:16,596 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 880bccb4d27e7b9d644e6b109920c175, disabling compactions & flushes 2024-11-12T12:26:16,596 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:16,596 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:16,596 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. after waiting 0 ms 2024-11-12T12:26:16,596 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:16,596 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 880bccb4d27e7b9d644e6b109920c175 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-12T12:26:16,602 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112708a3a9cc43744c0a1d2b468ed5eda15_880bccb4d27e7b9d644e6b109920c175 is 50, key is test_row_0/A:col10/1731414376432/Put/seqid=0 2024-11-12T12:26:16,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742254_1430 (size=12454) 2024-11-12T12:26:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T12:26:17,005 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:17,009 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112708a3a9cc43744c0a1d2b468ed5eda15_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112708a3a9cc43744c0a1d2b468ed5eda15_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:17,009 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/6b9f5c0195a041adaa94efa38d64ce37, store: [table=TestAcidGuarantees family=A region=880bccb4d27e7b9d644e6b109920c175] 2024-11-12T12:26:17,010 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/6b9f5c0195a041adaa94efa38d64ce37 is 175, key is test_row_0/A:col10/1731414376432/Put/seqid=0 2024-11-12T12:26:17,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742255_1431 (size=31255) 2024-11-12T12:26:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T12:26:17,413 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/6b9f5c0195a041adaa94efa38d64ce37 2024-11-12T12:26:17,419 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/8a27c0a28303475585ff4424f0695b00 is 50, key is test_row_0/B:col10/1731414376432/Put/seqid=0 2024-11-12T12:26:17,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742256_1432 (size=12301) 2024-11-12T12:26:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T12:26:17,823 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/8a27c0a28303475585ff4424f0695b00 2024-11-12T12:26:17,829 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/d87f307dafb74cf6bba65cbd0f4eda8f is 50, key is test_row_0/C:col10/1731414376432/Put/seqid=0 2024-11-12T12:26:17,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742257_1433 (size=12301) 2024-11-12T12:26:18,232 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/d87f307dafb74cf6bba65cbd0f4eda8f 2024-11-12T12:26:18,236 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/A/6b9f5c0195a041adaa94efa38d64ce37 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/6b9f5c0195a041adaa94efa38d64ce37 2024-11-12T12:26:18,239 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/6b9f5c0195a041adaa94efa38d64ce37, entries=150, sequenceid=340, filesize=30.5 K 2024-11-12T12:26:18,240 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/B/8a27c0a28303475585ff4424f0695b00 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/8a27c0a28303475585ff4424f0695b00 2024-11-12T12:26:18,243 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/8a27c0a28303475585ff4424f0695b00, entries=150, sequenceid=340, filesize=12.0 K 2024-11-12T12:26:18,243 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/.tmp/C/d87f307dafb74cf6bba65cbd0f4eda8f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d87f307dafb74cf6bba65cbd0f4eda8f 2024-11-12T12:26:18,246 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d87f307dafb74cf6bba65cbd0f4eda8f, entries=150, sequenceid=340, filesize=12.0 K 2024-11-12T12:26:18,247 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 880bccb4d27e7b9d644e6b109920c175 in 1651ms, sequenceid=340, compaction requested=false 2024-11-12T12:26:18,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/effdf7bba9cf4314bef5d6e29026b72c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3b46c98878e349f9b60c851778013f71, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/631ef8191f084c66bdbd8ba4804799a6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/978cf1f9b62a4485a7ba0d6d8e584c56, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/36fe1497156d40628bfe1cdf70f9fda2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/1f4bbf8337174a75b8bb06604c6d6e11, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/90f2f1c8d8d94bf4b49f6cb6acb4d033, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/64341b3ce6f54ebb8bce960c165ecbbc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2fa8fddc847f42e4b9b2cd3349559768, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/8f444ca6964a4c728a3ce9b227039669, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/371a2c41ee3142798a0e9c0cb8706344, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/052f38086f154fd8bf79efcfc92a5a0c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2676d708ee2e4b4d8e704c73c926d3d3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/029ea7891ca74eb19d447443f6b121b1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/0637212efc2d4cc891d74477eb035e31, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/81ab2b5ef5654509b32a1cfdf4e35fd1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/a8b1a5dd3b4248a18328ae2053b093b7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/7adeeb23cc84472da5fb6c229b6f28cf, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/bfe5705bd064471abab7aa21e576fa79, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/5b9a53e7b2624b0b9c9b990a51edbe0d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/dc5c3999a6f14155afc3fdd6c330f6ae, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/4e095bd4ede749488d8e9a54782d1ac5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/410be8630a194b5db20a00c7ac7362e4] to archive 2024-11-12T12:26:18,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:26:18,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/effdf7bba9cf4314bef5d6e29026b72c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/effdf7bba9cf4314bef5d6e29026b72c 2024-11-12T12:26:18,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3b46c98878e349f9b60c851778013f71 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3b46c98878e349f9b60c851778013f71 2024-11-12T12:26:18,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/631ef8191f084c66bdbd8ba4804799a6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/631ef8191f084c66bdbd8ba4804799a6 2024-11-12T12:26:18,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/978cf1f9b62a4485a7ba0d6d8e584c56 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/978cf1f9b62a4485a7ba0d6d8e584c56 2024-11-12T12:26:18,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/36fe1497156d40628bfe1cdf70f9fda2 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/36fe1497156d40628bfe1cdf70f9fda2 2024-11-12T12:26:18,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/1f4bbf8337174a75b8bb06604c6d6e11 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/1f4bbf8337174a75b8bb06604c6d6e11 2024-11-12T12:26:18,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/90f2f1c8d8d94bf4b49f6cb6acb4d033 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/90f2f1c8d8d94bf4b49f6cb6acb4d033 2024-11-12T12:26:18,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/64341b3ce6f54ebb8bce960c165ecbbc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/64341b3ce6f54ebb8bce960c165ecbbc 2024-11-12T12:26:18,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2fa8fddc847f42e4b9b2cd3349559768 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2fa8fddc847f42e4b9b2cd3349559768 2024-11-12T12:26:18,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/8f444ca6964a4c728a3ce9b227039669 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/8f444ca6964a4c728a3ce9b227039669 2024-11-12T12:26:18,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/371a2c41ee3142798a0e9c0cb8706344 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/371a2c41ee3142798a0e9c0cb8706344 2024-11-12T12:26:18,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/052f38086f154fd8bf79efcfc92a5a0c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/052f38086f154fd8bf79efcfc92a5a0c 2024-11-12T12:26:18,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2676d708ee2e4b4d8e704c73c926d3d3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/2676d708ee2e4b4d8e704c73c926d3d3 2024-11-12T12:26:18,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/029ea7891ca74eb19d447443f6b121b1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/029ea7891ca74eb19d447443f6b121b1 2024-11-12T12:26:18,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/0637212efc2d4cc891d74477eb035e31 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/0637212efc2d4cc891d74477eb035e31 2024-11-12T12:26:18,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/81ab2b5ef5654509b32a1cfdf4e35fd1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/81ab2b5ef5654509b32a1cfdf4e35fd1 2024-11-12T12:26:18,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/a8b1a5dd3b4248a18328ae2053b093b7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/a8b1a5dd3b4248a18328ae2053b093b7 2024-11-12T12:26:18,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/7adeeb23cc84472da5fb6c229b6f28cf to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/7adeeb23cc84472da5fb6c229b6f28cf 2024-11-12T12:26:18,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/bfe5705bd064471abab7aa21e576fa79 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/bfe5705bd064471abab7aa21e576fa79 2024-11-12T12:26:18,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/5b9a53e7b2624b0b9c9b990a51edbe0d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/5b9a53e7b2624b0b9c9b990a51edbe0d 2024-11-12T12:26:18,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/dc5c3999a6f14155afc3fdd6c330f6ae to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/dc5c3999a6f14155afc3fdd6c330f6ae 2024-11-12T12:26:18,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/4e095bd4ede749488d8e9a54782d1ac5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/4e095bd4ede749488d8e9a54782d1ac5 2024-11-12T12:26:18,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/410be8630a194b5db20a00c7ac7362e4 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/410be8630a194b5db20a00c7ac7362e4 2024-11-12T12:26:18,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c8432abd020b4f7fb2490e8fa34a940d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/366278c4907d4cda82db0be307fc0a4a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d88c6d2b886e41ab84e875a064c14dc5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/de23996fc1e342d092bf7a0ada786bdf, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/286f80452a52406f9220c291e55a6293, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/40ce0ede03a049f59f84cca00844fbc3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/bf707c34387746ce8abea46d7fb3b23f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/208cb68d697f47048a55f74f5d0ac9d9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/289a78e6d940462fa6b12a8509dd164c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d806af049e1f4fb98424f20c27198bda, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3ed4e44555e74344a8b5a39aa1bf292e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/a77d53773f8749389237d1f8f811b462, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/1c84710ddbac4e57acd0a2710ed74d79, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3e668e7b9d9449fb9de7b373881aa209, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/98351f29b56844339c32bd6f8095f46e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/edbdf71e7c29445cbddc79e4f3a0eef9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f9a2ef9e61a244ba97320e34e5551687, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/0981994db1e6493db87953ed9a7c898a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f24a7a16c4f2495faa78b1fd4fdd572b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c5b4381b9b3c4c21a55b5eb766f7ca28, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2c4c2a0562b94610b107f1eb11568fef, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2a28fc087929482ab651f787ee5b9067, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/7358eb559b174af6a7bf7e61c35f7cbe] to archive 2024-11-12T12:26:18,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:26:18,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c8432abd020b4f7fb2490e8fa34a940d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c8432abd020b4f7fb2490e8fa34a940d 2024-11-12T12:26:18,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/366278c4907d4cda82db0be307fc0a4a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/366278c4907d4cda82db0be307fc0a4a 2024-11-12T12:26:18,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d88c6d2b886e41ab84e875a064c14dc5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d88c6d2b886e41ab84e875a064c14dc5 2024-11-12T12:26:18,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/de23996fc1e342d092bf7a0ada786bdf to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/de23996fc1e342d092bf7a0ada786bdf 2024-11-12T12:26:18,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/286f80452a52406f9220c291e55a6293 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/286f80452a52406f9220c291e55a6293 2024-11-12T12:26:18,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/40ce0ede03a049f59f84cca00844fbc3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/40ce0ede03a049f59f84cca00844fbc3 2024-11-12T12:26:18,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/bf707c34387746ce8abea46d7fb3b23f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/bf707c34387746ce8abea46d7fb3b23f 2024-11-12T12:26:18,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/208cb68d697f47048a55f74f5d0ac9d9 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/208cb68d697f47048a55f74f5d0ac9d9 2024-11-12T12:26:18,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/289a78e6d940462fa6b12a8509dd164c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/289a78e6d940462fa6b12a8509dd164c 2024-11-12T12:26:18,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d806af049e1f4fb98424f20c27198bda to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/d806af049e1f4fb98424f20c27198bda 2024-11-12T12:26:18,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3ed4e44555e74344a8b5a39aa1bf292e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3ed4e44555e74344a8b5a39aa1bf292e 2024-11-12T12:26:18,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/a77d53773f8749389237d1f8f811b462 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/a77d53773f8749389237d1f8f811b462 2024-11-12T12:26:18,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/1c84710ddbac4e57acd0a2710ed74d79 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/1c84710ddbac4e57acd0a2710ed74d79 2024-11-12T12:26:18,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3e668e7b9d9449fb9de7b373881aa209 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3e668e7b9d9449fb9de7b373881aa209 2024-11-12T12:26:18,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/98351f29b56844339c32bd6f8095f46e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/98351f29b56844339c32bd6f8095f46e 2024-11-12T12:26:18,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/edbdf71e7c29445cbddc79e4f3a0eef9 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/edbdf71e7c29445cbddc79e4f3a0eef9 2024-11-12T12:26:18,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f9a2ef9e61a244ba97320e34e5551687 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f9a2ef9e61a244ba97320e34e5551687 2024-11-12T12:26:18,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/0981994db1e6493db87953ed9a7c898a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/0981994db1e6493db87953ed9a7c898a 2024-11-12T12:26:18,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f24a7a16c4f2495faa78b1fd4fdd572b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/f24a7a16c4f2495faa78b1fd4fdd572b 2024-11-12T12:26:18,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c5b4381b9b3c4c21a55b5eb766f7ca28 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/c5b4381b9b3c4c21a55b5eb766f7ca28 2024-11-12T12:26:18,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2c4c2a0562b94610b107f1eb11568fef to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2c4c2a0562b94610b107f1eb11568fef 2024-11-12T12:26:18,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2a28fc087929482ab651f787ee5b9067 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/2a28fc087929482ab651f787ee5b9067 2024-11-12T12:26:18,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/7358eb559b174af6a7bf7e61c35f7cbe to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/7358eb559b174af6a7bf7e61c35f7cbe 2024-11-12T12:26:18,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3f43cd6c6533424db3c2dcba220d88a9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ab8bd9de287247d08f3c0041f8800971, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/4a1d217feec34001972d731c8d430d6a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6b9bc791e94e47f3ab6cf3f67474c10b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7fbbc8992fc4332ab303b6aca3c95cb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eebaa21a08184320ba15bca973c2a246, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/52d606b6f652495288d91963a5e7179e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/566d0257af0b426fb8650d194f53217f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d38d7535289b4f26b11d448b723ad694, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/2ac9254da4dc4b4b814d868da374bc07, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/8a338339e8cb4d36ac6d05ab8b4cd02d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6ef804326dd2479186706ee5b555192a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eae11136be9147ccbe6170f4cf7c0ba1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/dfea19b1ccec4f2a9b7ab36c5f6f8ffc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/779c41e92ac746bd995544961c273a56, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3c45a62568e746e0971d776f3e018adc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a6b28a1793b3456a953faf6cd8b38b36, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ba5392221eb149739187565d3087ded7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/099195b25224442da678db9ce297ffb8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/5b06cccaa82b45e99597416391c44d2f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7320f1acaa34b73bb06c29836237cb8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/f7ee26e233694cc2917d04d68ef11022, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/41b03e9588e24e458f85cc78c1e055a6] to archive 2024-11-12T12:26:18,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:26:18,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3f43cd6c6533424db3c2dcba220d88a9 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3f43cd6c6533424db3c2dcba220d88a9 2024-11-12T12:26:18,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ab8bd9de287247d08f3c0041f8800971 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ab8bd9de287247d08f3c0041f8800971 2024-11-12T12:26:18,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/4a1d217feec34001972d731c8d430d6a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/4a1d217feec34001972d731c8d430d6a 2024-11-12T12:26:18,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6b9bc791e94e47f3ab6cf3f67474c10b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6b9bc791e94e47f3ab6cf3f67474c10b 2024-11-12T12:26:18,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7fbbc8992fc4332ab303b6aca3c95cb to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7fbbc8992fc4332ab303b6aca3c95cb 2024-11-12T12:26:18,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eebaa21a08184320ba15bca973c2a246 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eebaa21a08184320ba15bca973c2a246 2024-11-12T12:26:18,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/52d606b6f652495288d91963a5e7179e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/52d606b6f652495288d91963a5e7179e 2024-11-12T12:26:18,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/566d0257af0b426fb8650d194f53217f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/566d0257af0b426fb8650d194f53217f 2024-11-12T12:26:18,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d38d7535289b4f26b11d448b723ad694 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d38d7535289b4f26b11d448b723ad694 2024-11-12T12:26:18,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/2ac9254da4dc4b4b814d868da374bc07 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/2ac9254da4dc4b4b814d868da374bc07 2024-11-12T12:26:18,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/8a338339e8cb4d36ac6d05ab8b4cd02d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/8a338339e8cb4d36ac6d05ab8b4cd02d 2024-11-12T12:26:18,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6ef804326dd2479186706ee5b555192a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/6ef804326dd2479186706ee5b555192a 2024-11-12T12:26:18,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eae11136be9147ccbe6170f4cf7c0ba1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/eae11136be9147ccbe6170f4cf7c0ba1 2024-11-12T12:26:18,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/dfea19b1ccec4f2a9b7ab36c5f6f8ffc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/dfea19b1ccec4f2a9b7ab36c5f6f8ffc 2024-11-12T12:26:18,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/779c41e92ac746bd995544961c273a56 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/779c41e92ac746bd995544961c273a56 2024-11-12T12:26:18,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3c45a62568e746e0971d776f3e018adc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/3c45a62568e746e0971d776f3e018adc 2024-11-12T12:26:18,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a6b28a1793b3456a953faf6cd8b38b36 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a6b28a1793b3456a953faf6cd8b38b36 2024-11-12T12:26:18,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ba5392221eb149739187565d3087ded7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/ba5392221eb149739187565d3087ded7 2024-11-12T12:26:18,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/099195b25224442da678db9ce297ffb8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/099195b25224442da678db9ce297ffb8 2024-11-12T12:26:18,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/5b06cccaa82b45e99597416391c44d2f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/5b06cccaa82b45e99597416391c44d2f 2024-11-12T12:26:18,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7320f1acaa34b73bb06c29836237cb8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/a7320f1acaa34b73bb06c29836237cb8 2024-11-12T12:26:18,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/f7ee26e233694cc2917d04d68ef11022 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/f7ee26e233694cc2917d04d68ef11022 2024-11-12T12:26:18,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/41b03e9588e24e458f85cc78c1e055a6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/41b03e9588e24e458f85cc78c1e055a6 2024-11-12T12:26:18,315 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/recovered.edits/343.seqid, newMaxSeqId=343, maxSeqId=4 2024-11-12T12:26:18,316 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. 2024-11-12T12:26:18,316 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 880bccb4d27e7b9d644e6b109920c175: Waiting for close lock at 1731414376596Running coprocessor pre-close hooks at 1731414376596Disabling compacts and flushes for region at 1731414376596Disabling writes for close at 1731414376596Obtaining lock to block concurrent updates at 1731414376596Preparing flush snapshotting stores in 880bccb4d27e7b9d644e6b109920c175 at 1731414376596Finished memstore snapshotting TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731414376596Flushing stores of TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175. at 1731414376597 (+1 ms)Flushing 880bccb4d27e7b9d644e6b109920c175/A: creating writer at 1731414376597Flushing mob file 880bccb4d27e7b9d644e6b109920c175/A: appending metadata at 1731414376602 (+5 ms)Flushing mob file 880bccb4d27e7b9d644e6b109920c175/A: closing flushed file at 1731414376602Flushing 880bccb4d27e7b9d644e6b109920c175/A: appending metadata at 1731414377010 (+408 ms)Flushing 880bccb4d27e7b9d644e6b109920c175/A: closing flushed file at 1731414377010Flushing 880bccb4d27e7b9d644e6b109920c175/B: creating writer at 1731414377417 (+407 ms)Flushing 880bccb4d27e7b9d644e6b109920c175/B: appending metadata at 1731414377419 (+2 ms)Flushing 880bccb4d27e7b9d644e6b109920c175/B: closing flushed file at 1731414377419Flushing 880bccb4d27e7b9d644e6b109920c175/C: creating writer at 1731414377826 (+407 ms)Flushing 880bccb4d27e7b9d644e6b109920c175/C: appending metadata at 1731414377828 (+2 ms)Flushing 880bccb4d27e7b9d644e6b109920c175/C: closing flushed file at 1731414377828Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@480f4aae: reopening flushed file at 1731414378236 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fa4d7fc: reopening flushed file at 1731414378239 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c892393: reopening flushed file at 1731414378243 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 880bccb4d27e7b9d644e6b109920c175 in 1651ms, sequenceid=340, compaction requested=false at 1731414378247 (+4 ms)Writing region close event to WAL at 1731414378313 (+66 ms)Running coprocessor post-close hooks at 1731414378316 (+3 ms)Closed at 1731414378316 2024-11-12T12:26:18,318 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=880bccb4d27e7b9d644e6b109920c175, regionState=CLOSED 2024-11-12T12:26:18,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:26:18,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-12T12:26:18,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 880bccb4d27e7b9d644e6b109920c175, server=4a91c05c96a4,38717,1731414257691 in 1.8770 sec 2024-11-12T12:26:18,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-12T12:26:18,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=880bccb4d27e7b9d644e6b109920c175, UNASSIGN in 1.8800 sec 2024-11-12T12:26:18,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-12T12:26:18,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8830 sec 2024-11-12T12:26:18,325 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414378325"}]},"ts":"1731414378325"} 2024-11-12T12:26:18,326 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T12:26:18,326 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T12:26:18,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8910 sec 2024-11-12T12:26:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T12:26:18,582 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:18,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T12:26:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:18,584 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T12:26:18,584 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:18,586 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,587 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/recovered.edits] 2024-11-12T12:26:18,589 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3f5b1cc132954257ba3e8402838f22ac to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/3f5b1cc132954257ba3e8402838f22ac 2024-11-12T12:26:18,590 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/6b9f5c0195a041adaa94efa38d64ce37 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/A/6b9f5c0195a041adaa94efa38d64ce37 2024-11-12T12:26:18,592 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3842d2b48cf5426dae7604642a3f6bbc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/3842d2b48cf5426dae7604642a3f6bbc 2024-11-12T12:26:18,593 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/8a27c0a28303475585ff4424f0695b00 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/B/8a27c0a28303475585ff4424f0695b00 2024-11-12T12:26:18,594 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d87f307dafb74cf6bba65cbd0f4eda8f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/d87f307dafb74cf6bba65cbd0f4eda8f 2024-11-12T12:26:18,595 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/f31ba5bd6d0d4a85b824ba9a9fa58113 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/C/f31ba5bd6d0d4a85b824ba9a9fa58113 2024-11-12T12:26:18,598 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/recovered.edits/343.seqid to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175/recovered.edits/343.seqid 2024-11-12T12:26:18,598 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,598 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T12:26:18,598 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T12:26:18,599 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T12:26:18,601 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120429f1642c4b44bc877105b4f8118d4a_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120429f1642c4b44bc877105b4f8118d4a_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,602 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112052a68527a414fb6be07fb9202ddc6e9_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112052a68527a414fb6be07fb9202ddc6e9_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,603 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111214e7af7d9777443985615bfba2aeeef6_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111214e7af7d9777443985615bfba2aeeef6_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,604 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121873954e384946898a17694026d10839_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121873954e384946898a17694026d10839_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,605 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123b70e710b0734ac4a228d1f0ed89fb49_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123b70e710b0734ac4a228d1f0ed89fb49_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,605 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123e8bfdb01e034caabd7a14d94ec9100e_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123e8bfdb01e034caabd7a14d94ec9100e_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,606 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125ba56a26217f416d8b8d4089f95416c0_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125ba56a26217f416d8b8d4089f95416c0_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,607 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112708a3a9cc43744c0a1d2b468ed5eda15_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112708a3a9cc43744c0a1d2b468ed5eda15_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,607 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111279e6352ef84845cfaa9dda14503342cd_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111279e6352ef84845cfaa9dda14503342cd_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,608 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111289aed92136bd49c9a958577af5e74d37_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111289aed92136bd49c9a958577af5e74d37_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,609 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128e53f6e970ad478fae93515042fbe114_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128e53f6e970ad478fae93515042fbe114_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,609 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129f3e2f7d7ab84c589c1c272a1ac2002b_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129f3e2f7d7ab84c589c1c272a1ac2002b_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,610 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b7b6a36734d647b091349d02da84bc65_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b7b6a36734d647b091349d02da84bc65_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,611 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bab098699f4d4715a68e82675558d144_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bab098699f4d4715a68e82675558d144_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,612 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c3a89be2a80a4ff7aa8dc4b1e46e1c40_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c3a89be2a80a4ff7aa8dc4b1e46e1c40_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,612 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cd9d8793e0ca4ac9af84f2b1ce3f8cf4_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cd9d8793e0ca4ac9af84f2b1ce3f8cf4_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,613 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e667540437754f4ea4feec7a37c394a5_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e667540437754f4ea4feec7a37c394a5_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,614 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f80311f8bca34a06ad1b3b806dfa1297_880bccb4d27e7b9d644e6b109920c175 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f80311f8bca34a06ad1b3b806dfa1297_880bccb4d27e7b9d644e6b109920c175 2024-11-12T12:26:18,614 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T12:26:18,616 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:18,617 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T12:26:18,619 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T12:26:18,620 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:18,620 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T12:26:18,620 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731414378620"}]},"ts":"9223372036854775807"} 2024-11-12T12:26:18,621 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T12:26:18,621 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 880bccb4d27e7b9d644e6b109920c175, NAME => 'TestAcidGuarantees,,1731414343972.880bccb4d27e7b9d644e6b109920c175.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T12:26:18,621 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T12:26:18,621 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731414378621"}]},"ts":"9223372036854775807"} 2024-11-12T12:26:18,622 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T12:26:18,623 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:18,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-12T12:26:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T12:26:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T12:26:18,694 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:18,699 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 240), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=520 (was 569), ProcessCount=11 (was 11), AvailableMemoryMB=6733 (was 6771) 2024-11-12T12:26:18,706 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=520, ProcessCount=11, AvailableMemoryMB=6734 2024-11-12T12:26:18,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T12:26:18,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:26:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:18,709 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T12:26:18,709 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:18,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-12T12:26:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T12:26:18,710 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T12:26:18,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742258_1434 (size=959) 2024-11-12T12:26:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T12:26:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T12:26:19,116 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a 2024-11-12T12:26:19,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742259_1435 (size=53) 2024-11-12T12:26:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T12:26:19,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:26:19,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c64ea0949ce45a18c8de4f8e776f777e, disabling compactions & flushes 2024-11-12T12:26:19,522 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:19,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:19,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. after waiting 0 ms 2024-11-12T12:26:19,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:19,522 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:19,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c64ea0949ce45a18c8de4f8e776f777e: Waiting for close lock at 1731414379521Disabling compacts and flushes for region at 1731414379521Disabling writes for close at 1731414379522 (+1 ms)Writing region close event to WAL at 1731414379522Closed at 1731414379522 2024-11-12T12:26:19,523 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T12:26:19,523 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731414379523"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731414379523"}]},"ts":"1731414379523"} 2024-11-12T12:26:19,524 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T12:26:19,525 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T12:26:19,525 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414379525"}]},"ts":"1731414379525"} 2024-11-12T12:26:19,526 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T12:26:19,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c64ea0949ce45a18c8de4f8e776f777e, ASSIGN}] 2024-11-12T12:26:19,527 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c64ea0949ce45a18c8de4f8e776f777e, ASSIGN 2024-11-12T12:26:19,528 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c64ea0949ce45a18c8de4f8e776f777e, ASSIGN; state=OFFLINE, location=4a91c05c96a4,38717,1731414257691; forceNewPlan=false, retain=false 2024-11-12T12:26:19,679 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=c64ea0949ce45a18c8de4f8e776f777e, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:19,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c64ea0949ce45a18c8de4f8e776f777e, ASSIGN because future has completed 2024-11-12T12:26:19,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:26:19,835 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:19,835 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:26:19,835 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,835 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:26:19,835 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,835 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,836 INFO [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,838 INFO [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c64ea0949ce45a18c8de4f8e776f777e columnFamilyName A 2024-11-12T12:26:19,838 DEBUG [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:19,838 INFO [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] regionserver.HStore(327): Store=c64ea0949ce45a18c8de4f8e776f777e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:26:19,838 INFO [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,839 INFO [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c64ea0949ce45a18c8de4f8e776f777e columnFamilyName B 2024-11-12T12:26:19,839 DEBUG [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:19,840 INFO [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] regionserver.HStore(327): Store=c64ea0949ce45a18c8de4f8e776f777e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:26:19,840 INFO [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,841 INFO [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c64ea0949ce45a18c8de4f8e776f777e columnFamilyName C 2024-11-12T12:26:19,841 DEBUG [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:19,841 INFO [StoreOpener-c64ea0949ce45a18c8de4f8e776f777e-1 {}] regionserver.HStore(327): Store=c64ea0949ce45a18c8de4f8e776f777e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:26:19,841 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,842 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,842 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,842 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,842 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,843 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:26:19,844 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,845 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T12:26:19,845 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened c64ea0949ce45a18c8de4f8e776f777e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66614472, jitterRate=-0.0073670148849487305}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:26:19,845 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:19,846 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for c64ea0949ce45a18c8de4f8e776f777e: Running coprocessor pre-open hook at 1731414379835Writing region info on filesystem at 1731414379835Initializing all the Stores at 1731414379836 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414379836Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414379836Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414379836Cleaning up temporary data from old regions at 1731414379843 (+7 ms)Running coprocessor post-open hooks at 1731414379845 (+2 ms)Region opened successfully at 1731414379846 (+1 ms) 2024-11-12T12:26:19,846 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., pid=120, masterSystemTime=1731414379832 2024-11-12T12:26:19,848 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:19,848 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:19,848 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=c64ea0949ce45a18c8de4f8e776f777e, regionState=OPEN, openSeqNum=2, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:19,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:26:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T12:26:19,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-12T12:26:19,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 in 169 msec 2024-11-12T12:26:19,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-12T12:26:19,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c64ea0949ce45a18c8de4f8e776f777e, ASSIGN in 325 msec 2024-11-12T12:26:19,854 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T12:26:19,854 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414379854"}]},"ts":"1731414379854"} 2024-11-12T12:26:19,855 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T12:26:19,855 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T12:26:19,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-12T12:26:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-12T12:26:20,862 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:20,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,864 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a7e74e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,866 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6a7d8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,866 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,869 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T12:26:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a91c05c96a4,41939,1731414256942 2024-11-12T12:26:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@11c25c98 2024-11-12T12:26:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T12:26:20,870 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T12:26:20,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,872 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4070311b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,873 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258e0c53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,874 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1d0f2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d8e64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,876 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a97ac3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,877 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563cca06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,879 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2331e76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,880 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5322478e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,881 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3183debf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,882 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c0fa3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f25159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2063f0e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,887 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d52dfbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,888 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f825a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d68b196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,891 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6735cf7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,893 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ae3a3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,894 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc03a90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:20,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:20,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@118072d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:20,898 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20928982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:20,899 DEBUG [Thread-1973 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,899 DEBUG [Thread-1958 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,900 DEBUG [Thread-1969 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,900 DEBUG [Thread-1960 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,900 DEBUG [Thread-1962 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,901 DEBUG [Thread-1956 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,902 DEBUG [Thread-1967 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,902 DEBUG [Thread-1975 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,903 DEBUG [Thread-1975 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,903 DEBUG [Thread-1975 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,903 DEBUG [Thread-1964 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,904 DEBUG [Thread-1964 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,904 DEBUG [Thread-1964 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,907 DEBUG [Thread-1971 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-12T12:26:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:20,907 DEBUG [Thread-1964 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T12:26:20,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:20,910 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2] 2024-11-12T12:26:20,912 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:20,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:20,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:26:20,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/35f9bb034f50418aa137e3876c25e136 is 50, key is test_row_0/A:col10/1731414380912/Put/seqid=0 2024-11-12T12:26:20,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742260_1436 (size=16681) 2024-11-12T12:26:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414440994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414440994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414441000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414441001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414441002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T12:26:21,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T12:26:21,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:21,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414441111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414441111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414441111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414441111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414441111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T12:26:21,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:21,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T12:26:21,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414441321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414441321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414441321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414441321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414441321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/35f9bb034f50418aa137e3876c25e136 2024-11-12T12:26:21,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/b8c7cf3e5bb4400dbedfbfdc71e986ec is 50, key is test_row_0/B:col10/1731414380912/Put/seqid=0 2024-11-12T12:26:21,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742261_1437 (size=12001) 2024-11-12T12:26:21,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/b8c7cf3e5bb4400dbedfbfdc71e986ec 2024-11-12T12:26:21,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T12:26:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/9443573e882a406f9c1e9cbb69eb2b52 is 50, key is test_row_0/C:col10/1731414380912/Put/seqid=0 2024-11-12T12:26:21,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742262_1438 (size=12001) 2024-11-12T12:26:21,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T12:26:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T12:26:21,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414441631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414441631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414441631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414441631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414441641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:21,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T12:26:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T12:26:21,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:21,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:21,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/9443573e882a406f9c1e9cbb69eb2b52 2024-11-12T12:26:21,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/35f9bb034f50418aa137e3876c25e136 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/35f9bb034f50418aa137e3876c25e136 2024-11-12T12:26:21,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/35f9bb034f50418aa137e3876c25e136, entries=250, sequenceid=14, filesize=16.3 K 2024-11-12T12:26:21,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/b8c7cf3e5bb4400dbedfbfdc71e986ec as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b8c7cf3e5bb4400dbedfbfdc71e986ec 2024-11-12T12:26:21,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b8c7cf3e5bb4400dbedfbfdc71e986ec, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T12:26:21,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/9443573e882a406f9c1e9cbb69eb2b52 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9443573e882a406f9c1e9cbb69eb2b52 2024-11-12T12:26:21,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9443573e882a406f9c1e9cbb69eb2b52, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T12:26:21,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c64ea0949ce45a18c8de4f8e776f777e in 945ms, sequenceid=14, compaction requested=false 2024-11-12T12:26:21,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:21,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T12:26:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:21,990 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-12T12:26:21,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/149348637b6b4d38abc926b55a9d6ee9 is 50, key is test_row_0/A:col10/1731414380983/Put/seqid=0 2024-11-12T12:26:22,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742263_1439 (size=12001) 2024-11-12T12:26:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T12:26:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:22,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:22,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414442157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414442158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414442159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414442161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414442165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414442271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414442271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414442271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414442271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,400 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/149348637b6b4d38abc926b55a9d6ee9 2024-11-12T12:26:22,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/28ef4fd2db4e46b087a14a97386d4dc2 is 50, key is test_row_0/B:col10/1731414380983/Put/seqid=0 2024-11-12T12:26:22,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742264_1440 (size=12001) 2024-11-12T12:26:22,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414442481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414442481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414442481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414442491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414442791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414442791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414442792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414442801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:22,813 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/28ef4fd2db4e46b087a14a97386d4dc2 2024-11-12T12:26:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/c794ae16c37b4eaf96c8cd1d2b224184 is 50, key is test_row_0/C:col10/1731414380983/Put/seqid=0 2024-11-12T12:26:22,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742265_1441 (size=12001) 2024-11-12T12:26:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T12:26:23,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414443181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,190 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:23,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,233 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/c794ae16c37b4eaf96c8cd1d2b224184 2024-11-12T12:26:23,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/149348637b6b4d38abc926b55a9d6ee9 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/149348637b6b4d38abc926b55a9d6ee9 2024-11-12T12:26:23,240 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/149348637b6b4d38abc926b55a9d6ee9, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T12:26:23,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/28ef4fd2db4e46b087a14a97386d4dc2 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/28ef4fd2db4e46b087a14a97386d4dc2 2024-11-12T12:26:23,244 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/28ef4fd2db4e46b087a14a97386d4dc2, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T12:26:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/c794ae16c37b4eaf96c8cd1d2b224184 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/c794ae16c37b4eaf96c8cd1d2b224184 2024-11-12T12:26:23,254 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/c794ae16c37b4eaf96c8cd1d2b224184, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T12:26:23,254 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c64ea0949ce45a18c8de4f8e776f777e in 1264ms, sequenceid=37, compaction requested=false 2024-11-12T12:26:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:23,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-12T12:26:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-12T12:26:23,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-12T12:26:23,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3440 sec 2024-11-12T12:26:23,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.3540 sec 2024-11-12T12:26:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:23,315 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:23,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-12T12:26:23,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/fb81bbde2db84b28a5817c5522aceacb is 50, key is test_row_0/A:col10/1731414382157/Put/seqid=0 2024-11-12T12:26:23,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742266_1442 (size=14341) 2024-11-12T12:26:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414443388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414443389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414443391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414443399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414443511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414443511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414443511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414443511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414443731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/fb81bbde2db84b28a5817c5522aceacb 2024-11-12T12:26:23,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414443731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414443731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414443731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:23,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/ae453f589f814397adfa755209adb8c1 is 50, key is test_row_0/B:col10/1731414382157/Put/seqid=0 2024-11-12T12:26:23,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742267_1443 (size=12001) 2024-11-12T12:26:23,836 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T12:26:24,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414444041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:24,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414444051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:24,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414444051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414444051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:24,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/ae453f589f814397adfa755209adb8c1 2024-11-12T12:26:24,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/9194c5f646954bff96372e8f51573502 is 50, key is test_row_0/C:col10/1731414382157/Put/seqid=0 2024-11-12T12:26:24,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742268_1444 (size=12001) 2024-11-12T12:26:24,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/9194c5f646954bff96372e8f51573502 2024-11-12T12:26:24,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/fb81bbde2db84b28a5817c5522aceacb as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb81bbde2db84b28a5817c5522aceacb 2024-11-12T12:26:24,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb81bbde2db84b28a5817c5522aceacb, entries=200, sequenceid=51, filesize=14.0 K 2024-11-12T12:26:24,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/ae453f589f814397adfa755209adb8c1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ae453f589f814397adfa755209adb8c1 2024-11-12T12:26:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414444561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414444561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414444561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414444561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:24,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ae453f589f814397adfa755209adb8c1, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T12:26:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/9194c5f646954bff96372e8f51573502 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9194c5f646954bff96372e8f51573502 2024-11-12T12:26:24,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9194c5f646954bff96372e8f51573502, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T12:26:24,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c64ea0949ce45a18c8de4f8e776f777e in 1262ms, sequenceid=51, compaction requested=true 2024-11-12T12:26:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:24,577 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:24,577 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:24,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:24,578 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:24,578 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:24,578 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/B is initiating minor compaction (all files) 2024-11-12T12:26:24,578 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/A is initiating minor compaction (all files) 2024-11-12T12:26:24,578 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/B in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:24,578 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/A in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:24,578 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b8c7cf3e5bb4400dbedfbfdc71e986ec, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/28ef4fd2db4e46b087a14a97386d4dc2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ae453f589f814397adfa755209adb8c1] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=35.2 K 2024-11-12T12:26:24,578 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/35f9bb034f50418aa137e3876c25e136, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/149348637b6b4d38abc926b55a9d6ee9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb81bbde2db84b28a5817c5522aceacb] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=42.0 K 2024-11-12T12:26:24,578 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35f9bb034f50418aa137e3876c25e136, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731414380907 2024-11-12T12:26:24,578 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting b8c7cf3e5bb4400dbedfbfdc71e986ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731414380912 2024-11-12T12:26:24,579 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 149348637b6b4d38abc926b55a9d6ee9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731414380983 2024-11-12T12:26:24,579 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ef4fd2db4e46b087a14a97386d4dc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731414380983 2024-11-12T12:26:24,579 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ae453f589f814397adfa755209adb8c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731414382157 2024-11-12T12:26:24,579 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb81bbde2db84b28a5817c5522aceacb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731414382157 2024-11-12T12:26:24,585 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#A#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:24,586 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/9ce836f730454d1f85e418878fd6e754 is 50, key is test_row_0/A:col10/1731414382157/Put/seqid=0 2024-11-12T12:26:24,586 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:24,586 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/5152fd61367e44bc86de4598cbbee017 is 50, key is test_row_0/B:col10/1731414382157/Put/seqid=0 2024-11-12T12:26:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742269_1445 (size=12104) 2024-11-12T12:26:24,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742270_1446 (size=12104) 2024-11-12T12:26:25,004 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/9ce836f730454d1f85e418878fd6e754 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/9ce836f730454d1f85e418878fd6e754 2024-11-12T12:26:25,008 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/5152fd61367e44bc86de4598cbbee017 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/5152fd61367e44bc86de4598cbbee017 2024-11-12T12:26:25,013 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/B of c64ea0949ce45a18c8de4f8e776f777e into 5152fd61367e44bc86de4598cbbee017(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:25,013 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/A of c64ea0949ce45a18c8de4f8e776f777e into 9ce836f730454d1f85e418878fd6e754(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:25,013 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:25,013 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:25,013 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/B, priority=13, startTime=1731414384577; duration=0sec 2024-11-12T12:26:25,013 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/A, priority=13, startTime=1731414384577; duration=0sec 2024-11-12T12:26:25,014 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:25,014 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:B 2024-11-12T12:26:25,014 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:25,014 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:A 2024-11-12T12:26:25,014 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:25,015 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:25,015 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/C is initiating minor compaction (all files) 2024-11-12T12:26:25,015 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/C in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:25,015 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9443573e882a406f9c1e9cbb69eb2b52, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/c794ae16c37b4eaf96c8cd1d2b224184, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9194c5f646954bff96372e8f51573502] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=35.2 K 2024-11-12T12:26:25,015 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 9443573e882a406f9c1e9cbb69eb2b52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731414380912 2024-11-12T12:26:25,015 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting c794ae16c37b4eaf96c8cd1d2b224184, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731414380983 2024-11-12T12:26:25,016 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 9194c5f646954bff96372e8f51573502, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731414382157 2024-11-12T12:26:25,022 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#C#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:25,022 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/cd006a1ccbed4cd0aad5e7f26eb4e710 is 50, key is test_row_0/C:col10/1731414382157/Put/seqid=0 2024-11-12T12:26:25,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742271_1447 (size=12104) 2024-11-12T12:26:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T12:26:25,071 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:25,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-12T12:26:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T12:26:25,073 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:25,074 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:25,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T12:26:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:25,203 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:25,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-12T12:26:25,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/ab41e01b9bad4ac98ba4748b42cdaf4f is 50, key is test_row_0/A:col10/1731414383398/Put/seqid=0 2024-11-12T12:26:25,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742272_1448 (size=14341) 2024-11-12T12:26:25,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T12:26:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:25,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:25,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414445266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:25,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T12:26:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:25,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:25,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:25,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:25,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:25,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414445381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T12:26:25,435 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/cd006a1ccbed4cd0aad5e7f26eb4e710 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/cd006a1ccbed4cd0aad5e7f26eb4e710 2024-11-12T12:26:25,439 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/C of c64ea0949ce45a18c8de4f8e776f777e into cd006a1ccbed4cd0aad5e7f26eb4e710(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:25,439 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:25,439 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/C, priority=13, startTime=1731414384577; duration=0sec 2024-11-12T12:26:25,439 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:25,439 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:C 2024-11-12T12:26:25,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T12:26:25,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:25,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:25,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:25,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:25,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:25,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414445581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:25,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414445581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:25,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414445581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:25,590 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:25,590 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:25,591 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:25,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414445591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:25,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414445591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,599 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:25,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/ab41e01b9bad4ac98ba4748b42cdaf4f 2024-11-12T12:26:25,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/cac977dc129c457eae4d495e61037a2d is 50, key is test_row_0/B:col10/1731414383398/Put/seqid=0 2024-11-12T12:26:25,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742273_1449 (size=12001) 2024-11-12T12:26:25,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/cac977dc129c457eae4d495e61037a2d 2024-11-12T12:26:25,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/448348853eb14fdebdfb46de19fbe2b3 is 50, key is test_row_0/C:col10/1731414383398/Put/seqid=0 2024-11-12T12:26:25,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742274_1450 (size=12001) 2024-11-12T12:26:25,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/448348853eb14fdebdfb46de19fbe2b3 2024-11-12T12:26:25,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/ab41e01b9bad4ac98ba4748b42cdaf4f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/ab41e01b9bad4ac98ba4748b42cdaf4f 2024-11-12T12:26:25,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/ab41e01b9bad4ac98ba4748b42cdaf4f, entries=200, sequenceid=76, filesize=14.0 K 2024-11-12T12:26:25,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/cac977dc129c457eae4d495e61037a2d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cac977dc129c457eae4d495e61037a2d 2024-11-12T12:26:25,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cac977dc129c457eae4d495e61037a2d, entries=150, sequenceid=76, filesize=11.7 K 2024-11-12T12:26:25,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/448348853eb14fdebdfb46de19fbe2b3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/448348853eb14fdebdfb46de19fbe2b3 2024-11-12T12:26:25,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/448348853eb14fdebdfb46de19fbe2b3, entries=150, sequenceid=76, filesize=11.7 K 2024-11-12T12:26:25,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c64ea0949ce45a18c8de4f8e776f777e in 470ms, sequenceid=76, compaction requested=false 2024-11-12T12:26:25,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:25,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T12:26:25,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:25,684 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-12T12:26:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/f4cadacb5d804062821c598c5113f544 is 50, key is test_row_0/A:col10/1731414385263/Put/seqid=0 2024-11-12T12:26:25,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742275_1451 (size=12001) 2024-11-12T12:26:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T12:26:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:25,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414446037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:26,092 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/f4cadacb5d804062821c598c5113f544 2024-11-12T12:26:26,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/47d6f1c932274f288853e5327ce42d27 is 50, key is test_row_0/B:col10/1731414385263/Put/seqid=0 2024-11-12T12:26:26,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742276_1452 (size=12001) 2024-11-12T12:26:26,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414446141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T12:26:26,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414446351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:26,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:26,501 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/47d6f1c932274f288853e5327ce42d27 2024-11-12T12:26:26,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/dfb0923db70643ebac1b1c729c5ca948 is 50, key is test_row_0/C:col10/1731414385263/Put/seqid=0 2024-11-12T12:26:26,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742277_1453 (size=12001) 2024-11-12T12:26:26,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414446661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:26,911 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/dfb0923db70643ebac1b1c729c5ca948 2024-11-12T12:26:26,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/f4cadacb5d804062821c598c5113f544 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/f4cadacb5d804062821c598c5113f544 2024-11-12T12:26:26,917 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/f4cadacb5d804062821c598c5113f544, entries=150, sequenceid=90, filesize=11.7 K 2024-11-12T12:26:26,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/47d6f1c932274f288853e5327ce42d27 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/47d6f1c932274f288853e5327ce42d27 2024-11-12T12:26:26,923 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/47d6f1c932274f288853e5327ce42d27, entries=150, sequenceid=90, filesize=11.7 K 2024-11-12T12:26:26,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/dfb0923db70643ebac1b1c729c5ca948 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfb0923db70643ebac1b1c729c5ca948 2024-11-12T12:26:26,928 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfb0923db70643ebac1b1c729c5ca948, entries=150, sequenceid=90, filesize=11.7 K 2024-11-12T12:26:26,929 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c64ea0949ce45a18c8de4f8e776f777e in 1246ms, sequenceid=90, compaction requested=true 2024-11-12T12:26:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-12T12:26:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-12T12:26:26,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-12T12:26:26,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8560 sec 2024-11-12T12:26:26,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8600 sec 2024-11-12T12:26:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:27,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-12T12:26:27,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/cd5997378ebd402a88e063daf2558755 is 50, key is test_row_0/A:col10/1731414385997/Put/seqid=0 2024-11-12T12:26:27,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742278_1454 (size=14341) 2024-11-12T12:26:27,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/cd5997378ebd402a88e063daf2558755 2024-11-12T12:26:27,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/04306de340cf48bda67b3bc598bed4be is 50, key is test_row_0/B:col10/1731414385997/Put/seqid=0 2024-11-12T12:26:27,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742279_1455 (size=12001) 2024-11-12T12:26:27,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/04306de340cf48bda67b3bc598bed4be 2024-11-12T12:26:27,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3b96956b2d124b5b830472d0b83aa6e0 is 50, key is test_row_0/C:col10/1731414385997/Put/seqid=0 2024-11-12T12:26:27,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742280_1456 (size=12001) 2024-11-12T12:26:27,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3b96956b2d124b5b830472d0b83aa6e0 2024-11-12T12:26:27,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/cd5997378ebd402a88e063daf2558755 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/cd5997378ebd402a88e063daf2558755 2024-11-12T12:26:27,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/cd5997378ebd402a88e063daf2558755, entries=200, sequenceid=114, filesize=14.0 K 2024-11-12T12:26:27,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/04306de340cf48bda67b3bc598bed4be as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/04306de340cf48bda67b3bc598bed4be 2024-11-12T12:26:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T12:26:27,222 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:27,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:27,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/04306de340cf48bda67b3bc598bed4be, entries=150, sequenceid=114, filesize=11.7 K 2024-11-12T12:26:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-12T12:26:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T12:26:27,224 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:27,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3b96956b2d124b5b830472d0b83aa6e0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3b96956b2d124b5b830472d0b83aa6e0 2024-11-12T12:26:27,225 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:27,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:27,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3b96956b2d124b5b830472d0b83aa6e0, entries=150, sequenceid=114, filesize=11.7 K 2024-11-12T12:26:27,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=40.25 KB/41220 for c64ea0949ce45a18c8de4f8e776f777e in 50ms, sequenceid=114, compaction requested=true 2024-11-12T12:26:27,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:27,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:27,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:27,229 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:27,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:27,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:27,229 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:27,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:27,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:27,232 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:27,232 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/B is initiating minor compaction (all files) 2024-11-12T12:26:27,232 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/B in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,232 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/5152fd61367e44bc86de4598cbbee017, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cac977dc129c457eae4d495e61037a2d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/47d6f1c932274f288853e5327ce42d27, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/04306de340cf48bda67b3bc598bed4be] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=47.0 K 2024-11-12T12:26:27,233 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 5152fd61367e44bc86de4598cbbee017, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731414382157 2024-11-12T12:26:27,233 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:27,233 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/A is initiating minor compaction (all files) 2024-11-12T12:26:27,233 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/A in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,233 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting cac977dc129c457eae4d495e61037a2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731414383387 2024-11-12T12:26:27,233 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/9ce836f730454d1f85e418878fd6e754, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/ab41e01b9bad4ac98ba4748b42cdaf4f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/f4cadacb5d804062821c598c5113f544, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/cd5997378ebd402a88e063daf2558755] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=51.5 K 2024-11-12T12:26:27,233 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d6f1c932274f288853e5327ce42d27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731414385253 2024-11-12T12:26:27,233 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ce836f730454d1f85e418878fd6e754, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731414382157 2024-11-12T12:26:27,233 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 04306de340cf48bda67b3bc598bed4be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731414385997 2024-11-12T12:26:27,233 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab41e01b9bad4ac98ba4748b42cdaf4f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731414383387 2024-11-12T12:26:27,233 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4cadacb5d804062821c598c5113f544, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731414385253 2024-11-12T12:26:27,234 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd5997378ebd402a88e063daf2558755, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731414385997 2024-11-12T12:26:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:27,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:27,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:26:27,251 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:27,252 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/6b8f6ed3e9d5438f8b2280907ac05c8a is 50, key is test_row_0/B:col10/1731414385997/Put/seqid=0 2024-11-12T12:26:27,254 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#A#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:27,254 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/466859199b394c33ba5c3a969d0deed0 is 50, key is test_row_0/A:col10/1731414385997/Put/seqid=0 2024-11-12T12:26:27,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/0293703c6cca49fe902a14452f668c83 is 50, key is test_row_0/A:col10/1731414387237/Put/seqid=0 2024-11-12T12:26:27,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742282_1458 (size=12241) 2024-11-12T12:26:27,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742283_1459 (size=11997) 2024-11-12T12:26:27,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/0293703c6cca49fe902a14452f668c83 2024-11-12T12:26:27,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742281_1457 (size=12241) 2024-11-12T12:26:27,290 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/6b8f6ed3e9d5438f8b2280907ac05c8a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/6b8f6ed3e9d5438f8b2280907ac05c8a 2024-11-12T12:26:27,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/7b26eff3a29c49a69f1f7db048fb28cd is 50, key is test_row_0/B:col10/1731414387237/Put/seqid=0 2024-11-12T12:26:27,298 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/B of c64ea0949ce45a18c8de4f8e776f777e into 6b8f6ed3e9d5438f8b2280907ac05c8a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:27,298 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:27,299 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/B, priority=12, startTime=1731414387229; duration=0sec 2024-11-12T12:26:27,299 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:27,299 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:B 2024-11-12T12:26:27,299 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:27,300 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:27,300 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/C is initiating minor compaction (all files) 2024-11-12T12:26:27,300 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/C in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,300 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/cd006a1ccbed4cd0aad5e7f26eb4e710, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/448348853eb14fdebdfb46de19fbe2b3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfb0923db70643ebac1b1c729c5ca948, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3b96956b2d124b5b830472d0b83aa6e0] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=47.0 K 2024-11-12T12:26:27,300 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting cd006a1ccbed4cd0aad5e7f26eb4e710, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731414382157 2024-11-12T12:26:27,300 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 448348853eb14fdebdfb46de19fbe2b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731414383387 2024-11-12T12:26:27,301 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb0923db70643ebac1b1c729c5ca948, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731414385253 2024-11-12T12:26:27,301 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b96956b2d124b5b830472d0b83aa6e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731414385997 2024-11-12T12:26:27,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742284_1460 (size=9657) 2024-11-12T12:26:27,314 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#C#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:27,315 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3de0b191b5d2419e904c7a04b3822e8c is 50, key is test_row_0/C:col10/1731414385997/Put/seqid=0 2024-11-12T12:26:27,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742285_1461 (size=12241) 2024-11-12T12:26:27,325 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3de0b191b5d2419e904c7a04b3822e8c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3de0b191b5d2419e904c7a04b3822e8c 2024-11-12T12:26:27,328 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/C of c64ea0949ce45a18c8de4f8e776f777e into 3de0b191b5d2419e904c7a04b3822e8c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:27,328 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:27,328 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/C, priority=12, startTime=1731414387229; duration=0sec 2024-11-12T12:26:27,328 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:27,328 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:C 2024-11-12T12:26:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T12:26:27,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T12:26:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414447415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:27,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:27,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414447521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:27,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T12:26:27,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:27,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T12:26:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414447601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414447602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:27,609 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:27,610 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:27,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:27,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:27,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414447611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:27,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414447611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:27,619 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:27,619 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:27,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T12:26:27,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:27,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,692 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/466859199b394c33ba5c3a969d0deed0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/466859199b394c33ba5c3a969d0deed0 2024-11-12T12:26:27,705 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/A of c64ea0949ce45a18c8de4f8e776f777e into 466859199b394c33ba5c3a969d0deed0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:27,705 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:27,705 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/A, priority=12, startTime=1731414387229; duration=0sec 2024-11-12T12:26:27,705 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:27,705 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:A 2024-11-12T12:26:27,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/7b26eff3a29c49a69f1f7db048fb28cd 2024-11-12T12:26:27,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/8a28f3358b7240658f8355d88597cc44 is 50, key is test_row_0/C:col10/1731414387237/Put/seqid=0 2024-11-12T12:26:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414447731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:27,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742286_1462 (size=9657) 2024-11-12T12:26:27,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T12:26:27,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:27,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T12:26:27,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T12:26:27,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:27,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:28,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414448041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:28,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/8a28f3358b7240658f8355d88597cc44 2024-11-12T12:26:28,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T12:26:28,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:28,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:28,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:28,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:28,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:28,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/0293703c6cca49fe902a14452f668c83 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/0293703c6cca49fe902a14452f668c83 2024-11-12T12:26:28,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/0293703c6cca49fe902a14452f668c83, entries=150, sequenceid=125, filesize=11.7 K 2024-11-12T12:26:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/7b26eff3a29c49a69f1f7db048fb28cd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7b26eff3a29c49a69f1f7db048fb28cd 2024-11-12T12:26:28,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7b26eff3a29c49a69f1f7db048fb28cd, entries=100, sequenceid=125, filesize=9.4 K 2024-11-12T12:26:28,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/8a28f3358b7240658f8355d88597cc44 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a28f3358b7240658f8355d88597cc44 2024-11-12T12:26:28,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a28f3358b7240658f8355d88597cc44, entries=100, sequenceid=125, filesize=9.4 K 2024-11-12T12:26:28,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c64ea0949ce45a18c8de4f8e776f777e in 908ms, sequenceid=125, compaction requested=false 2024-11-12T12:26:28,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:28,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T12:26:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:28,298 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:26:28,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/5705c6f0304d4ef28ae3b7ed5d346fe6 is 50, key is test_row_0/A:col10/1731414387401/Put/seqid=0 2024-11-12T12:26:28,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742287_1463 (size=12151) 2024-11-12T12:26:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T12:26:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:28,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:28,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414448586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:28,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:28,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:28,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:28,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414448701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:28,706 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/5705c6f0304d4ef28ae3b7ed5d346fe6 2024-11-12T12:26:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/c4f6463383984e53b83a8b71ecb8ec21 is 50, key is test_row_0/B:col10/1731414387401/Put/seqid=0 2024-11-12T12:26:28,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742288_1464 (size=12151) 2024-11-12T12:26:28,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414448911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:29,116 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/c4f6463383984e53b83a8b71ecb8ec21 2024-11-12T12:26:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/b35d17991bc549e88806c24b7706e16a is 50, key is test_row_0/C:col10/1731414387401/Put/seqid=0 2024-11-12T12:26:29,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742289_1465 (size=12151) 2024-11-12T12:26:29,125 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/b35d17991bc549e88806c24b7706e16a 2024-11-12T12:26:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/5705c6f0304d4ef28ae3b7ed5d346fe6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/5705c6f0304d4ef28ae3b7ed5d346fe6 2024-11-12T12:26:29,134 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/5705c6f0304d4ef28ae3b7ed5d346fe6, entries=150, sequenceid=153, filesize=11.9 K 2024-11-12T12:26:29,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/c4f6463383984e53b83a8b71ecb8ec21 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/c4f6463383984e53b83a8b71ecb8ec21 2024-11-12T12:26:29,138 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/c4f6463383984e53b83a8b71ecb8ec21, entries=150, sequenceid=153, filesize=11.9 K 2024-11-12T12:26:29,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/b35d17991bc549e88806c24b7706e16a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/b35d17991bc549e88806c24b7706e16a 2024-11-12T12:26:29,143 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/b35d17991bc549e88806c24b7706e16a, entries=150, sequenceid=153, filesize=11.9 K 2024-11-12T12:26:29,143 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c64ea0949ce45a18c8de4f8e776f777e in 846ms, sequenceid=153, compaction requested=true 2024-11-12T12:26:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-12T12:26:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-12T12:26:29,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-12T12:26:29,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9190 sec 2024-11-12T12:26:29,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9250 sec 2024-11-12T12:26:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:29,226 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:29,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-12T12:26:29,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/aea6833ebc8244999ac2394ecd40ea61 is 50, key is test_row_0/A:col10/1731414389225/Put/seqid=0 2024-11-12T12:26:29,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742290_1466 (size=14541) 2024-11-12T12:26:29,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414449362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T12:26:29,371 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:29,371 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-12T12:26:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T12:26:29,373 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:29,374 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:29,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:29,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414449471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T12:26:29,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:29,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:29,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:29,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:29,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/aea6833ebc8244999ac2394ecd40ea61 2024-11-12T12:26:29,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/987f9bbadd214d2281b855994b347cf3 is 50, key is test_row_0/B:col10/1731414389225/Put/seqid=0 2024-11-12T12:26:29,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742291_1467 (size=12151) 2024-11-12T12:26:29,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:29,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414449681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T12:26:29,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:29,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:29,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:29,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:29,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:29,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T12:26:30,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414450001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:30,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:30,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:30,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:30,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/987f9bbadd214d2281b855994b347cf3 2024-11-12T12:26:30,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/62f136ac2162471d985c56897d57b86a is 50, key is test_row_0/C:col10/1731414389225/Put/seqid=0 2024-11-12T12:26:30,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742292_1468 (size=12151) 2024-11-12T12:26:30,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:30,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:30,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:30,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:30,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/62f136ac2162471d985c56897d57b86a 2024-11-12T12:26:30,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/aea6833ebc8244999ac2394ecd40ea61 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/aea6833ebc8244999ac2394ecd40ea61 2024-11-12T12:26:30,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/aea6833ebc8244999ac2394ecd40ea61, entries=200, sequenceid=165, filesize=14.2 K 2024-11-12T12:26:30,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/987f9bbadd214d2281b855994b347cf3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/987f9bbadd214d2281b855994b347cf3 2024-11-12T12:26:30,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/987f9bbadd214d2281b855994b347cf3, entries=150, sequenceid=165, filesize=11.9 K 2024-11-12T12:26:30,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/62f136ac2162471d985c56897d57b86a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62f136ac2162471d985c56897d57b86a 2024-11-12T12:26:30,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62f136ac2162471d985c56897d57b86a, entries=150, sequenceid=165, filesize=11.9 K 2024-11-12T12:26:30,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c64ea0949ce45a18c8de4f8e776f777e in 1242ms, sequenceid=165, compaction requested=true 2024-11-12T12:26:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:30,468 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:30,468 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:30,469 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50930 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:30,469 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/A is initiating minor compaction (all files) 2024-11-12T12:26:30,470 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/A in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,470 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/466859199b394c33ba5c3a969d0deed0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/0293703c6cca49fe902a14452f668c83, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/5705c6f0304d4ef28ae3b7ed5d346fe6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/aea6833ebc8244999ac2394ecd40ea61] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=49.7 K 2024-11-12T12:26:30,470 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:30,470 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/B is initiating minor compaction (all files) 2024-11-12T12:26:30,470 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/B in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,470 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/6b8f6ed3e9d5438f8b2280907ac05c8a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7b26eff3a29c49a69f1f7db048fb28cd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/c4f6463383984e53b83a8b71ecb8ec21, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/987f9bbadd214d2281b855994b347cf3] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=45.1 K 2024-11-12T12:26:30,470 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466859199b394c33ba5c3a969d0deed0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731414385997 2024-11-12T12:26:30,470 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8f6ed3e9d5438f8b2280907ac05c8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731414385997 2024-11-12T12:26:30,470 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0293703c6cca49fe902a14452f668c83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731414387193 2024-11-12T12:26:30,471 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5705c6f0304d4ef28ae3b7ed5d346fe6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731414387381 2024-11-12T12:26:30,471 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b26eff3a29c49a69f1f7db048fb28cd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731414387237 2024-11-12T12:26:30,471 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f6463383984e53b83a8b71ecb8ec21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731414387381 2024-11-12T12:26:30,471 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea6833ebc8244999ac2394ecd40ea61, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731414388562 2024-11-12T12:26:30,471 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 987f9bbadd214d2281b855994b347cf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731414388562 2024-11-12T12:26:30,477 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:30,477 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/354ad5b8d3cd419eb73bd6bffdf48083 is 50, key is test_row_0/B:col10/1731414389225/Put/seqid=0 2024-11-12T12:26:30,478 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#A#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:30,478 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/eea12c0eb3f44e85b23fd4f806b67cd3 is 50, key is test_row_0/A:col10/1731414389225/Put/seqid=0 2024-11-12T12:26:30,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742293_1469 (size=12527) 2024-11-12T12:26:30,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742294_1470 (size=12527) 2024-11-12T12:26:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T12:26:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:30,517 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:30,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:26:30,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/fb6efb82e53d40979e4a0379e8d6ae19 is 50, key is test_row_0/A:col10/1731414390516/Put/seqid=0 2024-11-12T12:26:30,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742295_1471 (size=14541) 2024-11-12T12:26:30,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414450568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:30,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414450681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:30,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:30,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:30,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,893 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/eea12c0eb3f44e85b23fd4f806b67cd3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/eea12c0eb3f44e85b23fd4f806b67cd3 2024-11-12T12:26:30,894 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/354ad5b8d3cd419eb73bd6bffdf48083 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/354ad5b8d3cd419eb73bd6bffdf48083 2024-11-12T12:26:30,897 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/A of c64ea0949ce45a18c8de4f8e776f777e into eea12c0eb3f44e85b23fd4f806b67cd3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:30,897 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:30,897 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/A, priority=12, startTime=1731414390468; duration=0sec 2024-11-12T12:26:30,897 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:30,897 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:A 2024-11-12T12:26:30,897 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:30,897 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/B of c64ea0949ce45a18c8de4f8e776f777e into 354ad5b8d3cd419eb73bd6bffdf48083(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:30,897 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:30,897 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/B, priority=12, startTime=1731414390468; duration=0sec 2024-11-12T12:26:30,897 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:30,897 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:B 2024-11-12T12:26:30,898 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:30,898 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/C is initiating minor compaction (all files) 2024-11-12T12:26:30,898 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/C in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,898 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3de0b191b5d2419e904c7a04b3822e8c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a28f3358b7240658f8355d88597cc44, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/b35d17991bc549e88806c24b7706e16a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62f136ac2162471d985c56897d57b86a] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=45.1 K 2024-11-12T12:26:30,898 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de0b191b5d2419e904c7a04b3822e8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731414385997 2024-11-12T12:26:30,899 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a28f3358b7240658f8355d88597cc44, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731414387237 2024-11-12T12:26:30,899 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting b35d17991bc549e88806c24b7706e16a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731414387381 2024-11-12T12:26:30,899 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f136ac2162471d985c56897d57b86a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731414388562 2024-11-12T12:26:30,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:30,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:30,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:30,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:30,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414450901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:30,911 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#C#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:30,911 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/eab902f74d4f4f6681a2af38ff60f272 is 50, key is test_row_0/C:col10/1731414389225/Put/seqid=0 2024-11-12T12:26:30,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/fb6efb82e53d40979e4a0379e8d6ae19 2024-11-12T12:26:30,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742296_1472 (size=12527) 2024-11-12T12:26:30,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/fd70ee2243e34490bc0ea090d3d73e24 is 50, key is test_row_0/B:col10/1731414390516/Put/seqid=0 2024-11-12T12:26:30,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742297_1473 (size=12151) 2024-11-12T12:26:31,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:31,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:31,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:31,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:31,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414451212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:31,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:31,328 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/eab902f74d4f4f6681a2af38ff60f272 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/eab902f74d4f4f6681a2af38ff60f272 2024-11-12T12:26:31,333 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/C of c64ea0949ce45a18c8de4f8e776f777e into eab902f74d4f4f6681a2af38ff60f272(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:31,333 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:31,333 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/C, priority=12, startTime=1731414390468; duration=0sec 2024-11-12T12:26:31,333 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:31,333 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:C 2024-11-12T12:26:31,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/fd70ee2243e34490bc0ea090d3d73e24 2024-11-12T12:26:31,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/66919bf4347543ecb38bd35a00ccfd5d is 50, key is test_row_0/C:col10/1731414390516/Put/seqid=0 2024-11-12T12:26:31,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742298_1474 (size=12151) 2024-11-12T12:26:31,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T12:26:31,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42154 deadline: 1731414451631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:31,634 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:31,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1731414451641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:31,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42156 deadline: 1731414451641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:31,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:31,646 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42192 deadline: 1731414451641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:31,646 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:31,647 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in c64ea0949ce45a18c8de4f8e776f777e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:31,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:31,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414451731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:31,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/66919bf4347543ecb38bd35a00ccfd5d 2024-11-12T12:26:31,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/fb6efb82e53d40979e4a0379e8d6ae19 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb6efb82e53d40979e4a0379e8d6ae19 2024-11-12T12:26:31,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb6efb82e53d40979e4a0379e8d6ae19, entries=200, sequenceid=190, filesize=14.2 K 2024-11-12T12:26:31,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/fd70ee2243e34490bc0ea090d3d73e24 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fd70ee2243e34490bc0ea090d3d73e24 2024-11-12T12:26:31,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fd70ee2243e34490bc0ea090d3d73e24, entries=150, sequenceid=190, filesize=11.9 K 2024-11-12T12:26:31,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/66919bf4347543ecb38bd35a00ccfd5d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/66919bf4347543ecb38bd35a00ccfd5d 2024-11-12T12:26:31,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/66919bf4347543ecb38bd35a00ccfd5d, entries=150, sequenceid=190, filesize=11.9 K 2024-11-12T12:26:31,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c64ea0949ce45a18c8de4f8e776f777e in 1249ms, sequenceid=190, compaction requested=false 2024-11-12T12:26:31,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:31,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T12:26:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:31,821 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:26:31,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/a8307ca93d5e48c5a2be2a25bb215e16 is 50, key is test_row_0/A:col10/1731414390543/Put/seqid=0 2024-11-12T12:26:31,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742299_1475 (size=12151) 2024-11-12T12:26:32,267 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/a8307ca93d5e48c5a2be2a25bb215e16 2024-11-12T12:26:32,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/b728cc835f264803b9c6e69662bc6139 is 50, key is test_row_0/B:col10/1731414390543/Put/seqid=0 2024-11-12T12:26:32,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742300_1476 (size=12151) 2024-11-12T12:26:32,678 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/b728cc835f264803b9c6e69662bc6139 2024-11-12T12:26:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/6db272dc66de47658e52b01779aecd29 is 50, key is test_row_0/C:col10/1731414390543/Put/seqid=0 2024-11-12T12:26:32,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742301_1477 (size=12151) 2024-11-12T12:26:32,705 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/6db272dc66de47658e52b01779aecd29 2024-11-12T12:26:32,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/a8307ca93d5e48c5a2be2a25bb215e16 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a8307ca93d5e48c5a2be2a25bb215e16 2024-11-12T12:26:32,712 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a8307ca93d5e48c5a2be2a25bb215e16, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T12:26:32,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/b728cc835f264803b9c6e69662bc6139 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b728cc835f264803b9c6e69662bc6139 2024-11-12T12:26:32,720 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b728cc835f264803b9c6e69662bc6139, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T12:26:32,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/6db272dc66de47658e52b01779aecd29 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/6db272dc66de47658e52b01779aecd29 2024-11-12T12:26:32,726 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/6db272dc66de47658e52b01779aecd29, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T12:26:32,726 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for c64ea0949ce45a18c8de4f8e776f777e in 905ms, sequenceid=204, compaction requested=true 2024-11-12T12:26:32,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:32,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:32,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-12T12:26:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-12T12:26:32,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-12T12:26:32,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3530 sec 2024-11-12T12:26:32,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.3580 sec 2024-11-12T12:26:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:32,778 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:32,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:26:32,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/6f503409bdc24ed19976d8f27e1e8ff9 is 50, key is test_row_0/A:col10/1731414392768/Put/seqid=0 2024-11-12T12:26:32,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742302_1478 (size=16931) 2024-11-12T12:26:32,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/6f503409bdc24ed19976d8f27e1e8ff9 2024-11-12T12:26:32,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/408e39a23dcd41e498bd13df113f53ce is 50, key is test_row_0/B:col10/1731414392768/Put/seqid=0 2024-11-12T12:26:32,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742303_1479 (size=12151) 2024-11-12T12:26:33,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414453028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:33,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414453151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:33,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/408e39a23dcd41e498bd13df113f53ce 2024-11-12T12:26:33,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/ed52e08db1df4ae59b6ee9100fd4bab2 is 50, key is test_row_0/C:col10/1731414392768/Put/seqid=0 2024-11-12T12:26:33,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742304_1480 (size=12151) 2024-11-12T12:26:33,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414453361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T12:26:33,542 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:33,543 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-12T12:26:33,546 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T12:26:33,547 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:33,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:33,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/ed52e08db1df4ae59b6ee9100fd4bab2 2024-11-12T12:26:33,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/6f503409bdc24ed19976d8f27e1e8ff9 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6f503409bdc24ed19976d8f27e1e8ff9 2024-11-12T12:26:33,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6f503409bdc24ed19976d8f27e1e8ff9, entries=250, sequenceid=215, filesize=16.5 K 2024-11-12T12:26:33,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/408e39a23dcd41e498bd13df113f53ce as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/408e39a23dcd41e498bd13df113f53ce 2024-11-12T12:26:33,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/408e39a23dcd41e498bd13df113f53ce, entries=150, sequenceid=215, filesize=11.9 K 2024-11-12T12:26:33,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/ed52e08db1df4ae59b6ee9100fd4bab2 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/ed52e08db1df4ae59b6ee9100fd4bab2 2024-11-12T12:26:33,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/ed52e08db1df4ae59b6ee9100fd4bab2, entries=150, sequenceid=215, filesize=11.9 K 2024-11-12T12:26:33,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c64ea0949ce45a18c8de4f8e776f777e in 851ms, sequenceid=215, compaction requested=true 2024-11-12T12:26:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:33,629 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T12:26:33,630 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:33,630 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56150 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:33,630 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:33,630 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/A is initiating minor compaction (all files) 2024-11-12T12:26:33,630 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/B is initiating minor compaction (all files) 2024-11-12T12:26:33,630 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/A in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:33,630 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/B in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:33,630 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/eea12c0eb3f44e85b23fd4f806b67cd3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb6efb82e53d40979e4a0379e8d6ae19, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a8307ca93d5e48c5a2be2a25bb215e16, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6f503409bdc24ed19976d8f27e1e8ff9] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=54.8 K 2024-11-12T12:26:33,630 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/354ad5b8d3cd419eb73bd6bffdf48083, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fd70ee2243e34490bc0ea090d3d73e24, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b728cc835f264803b9c6e69662bc6139, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/408e39a23dcd41e498bd13df113f53ce] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=47.8 K 2024-11-12T12:26:33,631 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting eea12c0eb3f44e85b23fd4f806b67cd3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731414388562 2024-11-12T12:26:33,631 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 354ad5b8d3cd419eb73bd6bffdf48083, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731414388562 2024-11-12T12:26:33,631 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6efb82e53d40979e4a0379e8d6ae19, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731414389339 2024-11-12T12:26:33,631 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd70ee2243e34490bc0ea090d3d73e24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731414389344 2024-11-12T12:26:33,631 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a8307ca93d5e48c5a2be2a25bb215e16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731414390543 2024-11-12T12:26:33,631 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting b728cc835f264803b9c6e69662bc6139, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731414390543 2024-11-12T12:26:33,631 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f503409bdc24ed19976d8f27e1e8ff9, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414392755 2024-11-12T12:26:33,631 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408e39a23dcd41e498bd13df113f53ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414392765 2024-11-12T12:26:33,644 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:33,645 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/788c0a8d855d491f8f261831fae6dca0 is 50, key is test_row_0/A:col10/1731414392768/Put/seqid=0 2024-11-12T12:26:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T12:26:33,655 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:33,655 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/3a9edcd61b1f41998ae9410ddb82ef82 is 50, key is test_row_0/B:col10/1731414392768/Put/seqid=0 2024-11-12T12:26:33,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742305_1481 (size=12663) 2024-11-12T12:26:33,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742306_1482 (size=12663) 2024-11-12T12:26:33,678 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/3a9edcd61b1f41998ae9410ddb82ef82 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/3a9edcd61b1f41998ae9410ddb82ef82 2024-11-12T12:26:33,684 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/B of c64ea0949ce45a18c8de4f8e776f777e into 3a9edcd61b1f41998ae9410ddb82ef82(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:33,684 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:33,684 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/B, priority=12, startTime=1731414393629; duration=0sec 2024-11-12T12:26:33,684 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:33,684 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:B 2024-11-12T12:26:33,684 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:33,687 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:33,687 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/C is initiating minor compaction (all files) 2024-11-12T12:26:33,687 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/C in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:33,687 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/eab902f74d4f4f6681a2af38ff60f272, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/66919bf4347543ecb38bd35a00ccfd5d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/6db272dc66de47658e52b01779aecd29, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/ed52e08db1df4ae59b6ee9100fd4bab2] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=47.8 K 2024-11-12T12:26:33,687 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting eab902f74d4f4f6681a2af38ff60f272, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731414388562 2024-11-12T12:26:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:33,687 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:33,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-12T12:26:33,687 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66919bf4347543ecb38bd35a00ccfd5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731414389344 2024-11-12T12:26:33,688 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6db272dc66de47658e52b01779aecd29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731414390543 2024-11-12T12:26:33,688 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed52e08db1df4ae59b6ee9100fd4bab2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414392765 2024-11-12T12:26:33,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/3f476690957e428ab84b4c8bfa774b58 is 50, key is test_row_0/A:col10/1731414393011/Put/seqid=0 2024-11-12T12:26:33,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T12:26:33,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:33,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:33,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:33,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:33,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:33,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742307_1483 (size=14541) 2024-11-12T12:26:33,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/3f476690957e428ab84b4c8bfa774b58 2024-11-12T12:26:33,715 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#C#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:33,715 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/1f4b43cd551c403396e9bd16ba42f86b is 50, key is test_row_0/C:col10/1731414392768/Put/seqid=0 2024-11-12T12:26:33,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742308_1484 (size=12663) 2024-11-12T12:26:33,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/fc251e67a1f44587805fcd920685434e is 50, key is test_row_0/B:col10/1731414393011/Put/seqid=0 2024-11-12T12:26:33,736 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/1f4b43cd551c403396e9bd16ba42f86b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/1f4b43cd551c403396e9bd16ba42f86b 2024-11-12T12:26:33,740 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/C of c64ea0949ce45a18c8de4f8e776f777e into 1f4b43cd551c403396e9bd16ba42f86b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:33,740 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:33,740 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/C, priority=12, startTime=1731414393629; duration=0sec 2024-11-12T12:26:33,740 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:33,740 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:C 2024-11-12T12:26:33,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742309_1485 (size=12151) 2024-11-12T12:26:33,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414453768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:33,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T12:26:33,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:33,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:33,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:33,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:33,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T12:26:33,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414453871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:33,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:33,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:34,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T12:26:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:34,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,072 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/788c0a8d855d491f8f261831fae6dca0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/788c0a8d855d491f8f261831fae6dca0 2024-11-12T12:26:34,076 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/A of c64ea0949ce45a18c8de4f8e776f777e into 788c0a8d855d491f8f261831fae6dca0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:34,076 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:34,076 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/A, priority=12, startTime=1731414393629; duration=0sec 2024-11-12T12:26:34,076 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:34,076 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:A 2024-11-12T12:26:34,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414454081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:34,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/fc251e67a1f44587805fcd920685434e 2024-11-12T12:26:34,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3faf53c8cd0a43429967518d76633fe3 is 50, key is test_row_0/C:col10/1731414393011/Put/seqid=0 2024-11-12T12:26:34,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742310_1486 (size=12151) 2024-11-12T12:26:34,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T12:26:34,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:34,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:34,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:34,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T12:26:34,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T12:26:34,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:34,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:34,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:34,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414454391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:34,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T12:26:34,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:34,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:34,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:34,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:34,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3faf53c8cd0a43429967518d76633fe3 2024-11-12T12:26:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/3f476690957e428ab84b4c8bfa774b58 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/3f476690957e428ab84b4c8bfa774b58 2024-11-12T12:26:34,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/3f476690957e428ab84b4c8bfa774b58, entries=200, sequenceid=242, filesize=14.2 K 2024-11-12T12:26:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/fc251e67a1f44587805fcd920685434e as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fc251e67a1f44587805fcd920685434e 2024-11-12T12:26:34,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fc251e67a1f44587805fcd920685434e, entries=150, sequenceid=242, filesize=11.9 K 2024-11-12T12:26:34,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3faf53c8cd0a43429967518d76633fe3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3faf53c8cd0a43429967518d76633fe3 2024-11-12T12:26:34,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3faf53c8cd0a43429967518d76633fe3, entries=150, sequenceid=242, filesize=11.9 K 2024-11-12T12:26:34,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c64ea0949ce45a18c8de4f8e776f777e in 889ms, sequenceid=242, compaction requested=false 2024-11-12T12:26:34,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:34,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T12:26:34,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:34,620 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-12T12:26:34,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/6c21a5af65b142d0905e48ffc04a5bbd is 50, key is test_row_0/A:col10/1731414393754/Put/seqid=0 2024-11-12T12:26:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742311_1487 (size=12151) 2024-11-12T12:26:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T12:26:34,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:35,027 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/6c21a5af65b142d0905e48ffc04a5bbd 2024-11-12T12:26:35,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/8e90f2c8f85442a6880e2e56cb5276f1 is 50, key is test_row_0/B:col10/1731414393754/Put/seqid=0 2024-11-12T12:26:35,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414455027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:35,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742312_1488 (size=12151) 2024-11-12T12:26:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:35,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414455141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:35,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414455351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:35,435 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/8e90f2c8f85442a6880e2e56cb5276f1 2024-11-12T12:26:35,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/8a7bb94f9a2f436682e8f54d0edb96e7 is 50, key is test_row_0/C:col10/1731414393754/Put/seqid=0 2024-11-12T12:26:35,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742313_1489 (size=12151) 2024-11-12T12:26:35,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414455661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T12:26:35,843 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/8a7bb94f9a2f436682e8f54d0edb96e7 2024-11-12T12:26:35,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/6c21a5af65b142d0905e48ffc04a5bbd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6c21a5af65b142d0905e48ffc04a5bbd 2024-11-12T12:26:35,848 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6c21a5af65b142d0905e48ffc04a5bbd, entries=150, sequenceid=254, filesize=11.9 K 2024-11-12T12:26:35,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/8e90f2c8f85442a6880e2e56cb5276f1 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/8e90f2c8f85442a6880e2e56cb5276f1 2024-11-12T12:26:35,851 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/8e90f2c8f85442a6880e2e56cb5276f1, entries=150, sequenceid=254, filesize=11.9 K 2024-11-12T12:26:35,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/8a7bb94f9a2f436682e8f54d0edb96e7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a7bb94f9a2f436682e8f54d0edb96e7 2024-11-12T12:26:35,854 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a7bb94f9a2f436682e8f54d0edb96e7, entries=150, sequenceid=254, filesize=11.9 K 2024-11-12T12:26:35,855 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for c64ea0949ce45a18c8de4f8e776f777e in 1235ms, sequenceid=254, compaction requested=true 2024-11-12T12:26:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-12T12:26:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-12T12:26:35,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-12T12:26:35,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3090 sec 2024-11-12T12:26:35,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.3140 sec 2024-11-12T12:26:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:36,175 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:36,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-12T12:26:36,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/c03152fd08054b34837b056d31789b29 is 50, key is test_row_0/A:col10/1731414395026/Put/seqid=0 2024-11-12T12:26:36,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742314_1490 (size=14741) 2024-11-12T12:26:36,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414456215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:36,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:36,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414456331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:36,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414456541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:36,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/c03152fd08054b34837b056d31789b29 2024-11-12T12:26:36,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/ec12e20acf574de9a45b44761f864877 is 50, key is test_row_0/B:col10/1731414395026/Put/seqid=0 2024-11-12T12:26:36,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742315_1491 (size=12301) 2024-11-12T12:26:36,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414456851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:36,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:36,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:36,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:36,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/ec12e20acf574de9a45b44761f864877 2024-11-12T12:26:37,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/0aa24b7b84d0486c9104a0c701bfd360 is 50, key is test_row_0/C:col10/1731414395026/Put/seqid=0 2024-11-12T12:26:37,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742316_1492 (size=12301) 2024-11-12T12:26:37,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/0aa24b7b84d0486c9104a0c701bfd360 2024-11-12T12:26:37,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/c03152fd08054b34837b056d31789b29 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/c03152fd08054b34837b056d31789b29 2024-11-12T12:26:37,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/c03152fd08054b34837b056d31789b29, entries=200, sequenceid=281, filesize=14.4 K 2024-11-12T12:26:37,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/ec12e20acf574de9a45b44761f864877 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ec12e20acf574de9a45b44761f864877 2024-11-12T12:26:37,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ec12e20acf574de9a45b44761f864877, entries=150, sequenceid=281, filesize=12.0 K 2024-11-12T12:26:37,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/0aa24b7b84d0486c9104a0c701bfd360 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0aa24b7b84d0486c9104a0c701bfd360 2024-11-12T12:26:37,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0aa24b7b84d0486c9104a0c701bfd360, entries=150, sequenceid=281, filesize=12.0 K 2024-11-12T12:26:37,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for c64ea0949ce45a18c8de4f8e776f777e in 851ms, sequenceid=281, compaction requested=true 2024-11-12T12:26:37,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:37,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:37,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:37,026 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:37,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:37,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:37,026 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:37,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:37,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:37,028 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:37,028 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/B is initiating minor compaction (all files) 2024-11-12T12:26:37,028 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/B in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:37,028 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/3a9edcd61b1f41998ae9410ddb82ef82, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fc251e67a1f44587805fcd920685434e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/8e90f2c8f85442a6880e2e56cb5276f1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ec12e20acf574de9a45b44761f864877] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=48.1 K 2024-11-12T12:26:37,029 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:37,029 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/A is initiating minor compaction (all files) 2024-11-12T12:26:37,029 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/A in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:37,029 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/788c0a8d855d491f8f261831fae6dca0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/3f476690957e428ab84b4c8bfa774b58, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6c21a5af65b142d0905e48ffc04a5bbd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/c03152fd08054b34837b056d31789b29] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=52.8 K 2024-11-12T12:26:37,029 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788c0a8d855d491f8f261831fae6dca0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414392765 2024-11-12T12:26:37,029 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a9edcd61b1f41998ae9410ddb82ef82, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414392765 2024-11-12T12:26:37,029 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f476690957e428ab84b4c8bfa774b58, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731414393000 2024-11-12T12:26:37,029 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting fc251e67a1f44587805fcd920685434e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731414393000 2024-11-12T12:26:37,029 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c21a5af65b142d0905e48ffc04a5bbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731414393739 2024-11-12T12:26:37,030 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e90f2c8f85442a6880e2e56cb5276f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731414393739 2024-11-12T12:26:37,030 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting c03152fd08054b34837b056d31789b29, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731414395007 2024-11-12T12:26:37,030 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting ec12e20acf574de9a45b44761f864877, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731414395007 2024-11-12T12:26:37,037 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#B#compaction#417 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:37,037 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#A#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:37,038 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/cfeadd534e1c4ee69ec784670a6b7617 is 50, key is test_row_0/B:col10/1731414395026/Put/seqid=0 2024-11-12T12:26:37,038 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/516739aa3d3b442a9c49d375abf80920 is 50, key is test_row_0/A:col10/1731414395026/Put/seqid=0 2024-11-12T12:26:37,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742318_1494 (size=12949) 2024-11-12T12:26:37,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742317_1493 (size=12949) 2024-11-12T12:26:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:37,373 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:37,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:26:37,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/d2d23f3b902d46b092263adb5e7de7c7 is 50, key is test_row_0/A:col10/1731414397372/Put/seqid=0 2024-11-12T12:26:37,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742319_1495 (size=14741) 2024-11-12T12:26:37,456 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/516739aa3d3b442a9c49d375abf80920 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/516739aa3d3b442a9c49d375abf80920 2024-11-12T12:26:37,456 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/cfeadd534e1c4ee69ec784670a6b7617 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cfeadd534e1c4ee69ec784670a6b7617 2024-11-12T12:26:37,460 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/A of c64ea0949ce45a18c8de4f8e776f777e into 516739aa3d3b442a9c49d375abf80920(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:37,460 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/B of c64ea0949ce45a18c8de4f8e776f777e into cfeadd534e1c4ee69ec784670a6b7617(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:37,460 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:37,460 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:37,460 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/B, priority=12, startTime=1731414397026; duration=0sec 2024-11-12T12:26:37,460 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/A, priority=12, startTime=1731414397026; duration=0sec 2024-11-12T12:26:37,460 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:37,460 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:37,460 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:A 2024-11-12T12:26:37,460 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:B 2024-11-12T12:26:37,460 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:37,461 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:37,461 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/C is initiating minor compaction (all files) 2024-11-12T12:26:37,461 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/C in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:37,461 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/1f4b43cd551c403396e9bd16ba42f86b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3faf53c8cd0a43429967518d76633fe3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a7bb94f9a2f436682e8f54d0edb96e7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0aa24b7b84d0486c9104a0c701bfd360] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=48.1 K 2024-11-12T12:26:37,461 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f4b43cd551c403396e9bd16ba42f86b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414392765 2024-11-12T12:26:37,461 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3faf53c8cd0a43429967518d76633fe3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731414393000 2024-11-12T12:26:37,462 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7bb94f9a2f436682e8f54d0edb96e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731414393739 2024-11-12T12:26:37,462 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa24b7b84d0486c9104a0c701bfd360, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731414395007 2024-11-12T12:26:37,467 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:37,468 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/62e68c35873b49afa4aeafc145d27670 is 50, key is test_row_0/C:col10/1731414395026/Put/seqid=0 2024-11-12T12:26:37,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742320_1496 (size=12949) 2024-11-12T12:26:37,474 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/62e68c35873b49afa4aeafc145d27670 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62e68c35873b49afa4aeafc145d27670 2024-11-12T12:26:37,477 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/C of c64ea0949ce45a18c8de4f8e776f777e into 62e68c35873b49afa4aeafc145d27670(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:37,477 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:37,477 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/C, priority=12, startTime=1731414397026; duration=0sec 2024-11-12T12:26:37,477 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:37,477 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:C 2024-11-12T12:26:37,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414457514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:37,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414457631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T12:26:37,721 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:37,722 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-12T12:26:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T12:26:37,723 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:37,723 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:37,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:37,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/d2d23f3b902d46b092263adb5e7de7c7 2024-11-12T12:26:37,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/7c5869140eb84ba2b6ee52535e7217a0 is 50, key is test_row_0/B:col10/1731414397372/Put/seqid=0 2024-11-12T12:26:37,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742321_1497 (size=12301) 2024-11-12T12:26:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T12:26:37,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414457841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:37,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T12:26:37,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:37,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:37,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:37,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:37,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T12:26:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:38,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T12:26:38,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414458151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:38,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T12:26:38,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:38,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:38,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:38,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/7c5869140eb84ba2b6ee52535e7217a0 2024-11-12T12:26:38,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/dfa26dc4c33d49858ef2fa7e52d9ad39 is 50, key is test_row_0/C:col10/1731414397372/Put/seqid=0 2024-11-12T12:26:38,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742322_1498 (size=12301) 2024-11-12T12:26:38,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T12:26:38,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:38,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:38,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:38,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T12:26:38,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T12:26:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:38,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:38,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/dfa26dc4c33d49858ef2fa7e52d9ad39 2024-11-12T12:26:38,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/d2d23f3b902d46b092263adb5e7de7c7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/d2d23f3b902d46b092263adb5e7de7c7 2024-11-12T12:26:38,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/d2d23f3b902d46b092263adb5e7de7c7, entries=200, sequenceid=292, filesize=14.4 K 2024-11-12T12:26:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/7c5869140eb84ba2b6ee52535e7217a0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7c5869140eb84ba2b6ee52535e7217a0 2024-11-12T12:26:38,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7c5869140eb84ba2b6ee52535e7217a0, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T12:26:38,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/dfa26dc4c33d49858ef2fa7e52d9ad39 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfa26dc4c33d49858ef2fa7e52d9ad39 2024-11-12T12:26:38,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfa26dc4c33d49858ef2fa7e52d9ad39, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T12:26:38,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c64ea0949ce45a18c8de4f8e776f777e in 1237ms, sequenceid=292, compaction requested=false 2024-11-12T12:26:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:38,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T12:26:38,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:38,638 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:26:38,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/99d6511d12a44d459bc6f3286a2157f0 is 50, key is test_row_0/A:col10/1731414397500/Put/seqid=0 2024-11-12T12:26:38,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742323_1499 (size=12301) 2024-11-12T12:26:38,644 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/99d6511d12a44d459bc6f3286a2157f0 2024-11-12T12:26:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/f3532c5c9fc4408f8c0d614264b0d027 is 50, key is test_row_0/B:col10/1731414397500/Put/seqid=0 2024-11-12T12:26:38,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742324_1500 (size=12301) 2024-11-12T12:26:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:38,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:38,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414458698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:38,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:38,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:38,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:38,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414458811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T12:26:39,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414459021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:39,053 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/f3532c5c9fc4408f8c0d614264b0d027 2024-11-12T12:26:39,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/761a4ea9aaed42ad8f434aa35f9fb623 is 50, key is test_row_0/C:col10/1731414397500/Put/seqid=0 2024-11-12T12:26:39,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742325_1501 (size=12301) 2024-11-12T12:26:39,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414459330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:39,462 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/761a4ea9aaed42ad8f434aa35f9fb623 2024-11-12T12:26:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/99d6511d12a44d459bc6f3286a2157f0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/99d6511d12a44d459bc6f3286a2157f0 2024-11-12T12:26:39,469 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/99d6511d12a44d459bc6f3286a2157f0, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T12:26:39,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/f3532c5c9fc4408f8c0d614264b0d027 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/f3532c5c9fc4408f8c0d614264b0d027 2024-11-12T12:26:39,472 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/f3532c5c9fc4408f8c0d614264b0d027, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T12:26:39,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/761a4ea9aaed42ad8f434aa35f9fb623 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/761a4ea9aaed42ad8f434aa35f9fb623 2024-11-12T12:26:39,475 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/761a4ea9aaed42ad8f434aa35f9fb623, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T12:26:39,476 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c64ea0949ce45a18c8de4f8e776f777e in 839ms, sequenceid=320, compaction requested=true 2024-11-12T12:26:39,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:39,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:39,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-12T12:26:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-12T12:26:39,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-12T12:26:39,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7530 sec 2024-11-12T12:26:39,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.7570 sec 2024-11-12T12:26:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:39,844 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:39,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-12T12:26:39,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/a7080f0f3f53445cbb0255de25431f5b is 50, key is test_row_0/A:col10/1731414398684/Put/seqid=0 2024-11-12T12:26:39,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742326_1502 (size=14741) 2024-11-12T12:26:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T12:26:39,871 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:39,871 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-12T12:26:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T12:26:39,872 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:39,873 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:39,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T12:26:39,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414459990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:40,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T12:26:40,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:40,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414460101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:40,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T12:26:40,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:40,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T12:26:40,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/a7080f0f3f53445cbb0255de25431f5b 2024-11-12T12:26:40,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/0cb7929e681e4a708a2a7788c699601d is 50, key is test_row_0/B:col10/1731414398684/Put/seqid=0 2024-11-12T12:26:40,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742327_1503 (size=12301) 2024-11-12T12:26:40,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414460311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:40,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T12:26:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T12:26:40,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:40,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T12:26:40,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T12:26:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42144 deadline: 1731414460630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 , the old value is region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., hostname=4a91c05c96a4,38717,1731414257691, seqNum=2 because the exception is null or not the one we care about 2024-11-12T12:26:40,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/0cb7929e681e4a708a2a7788c699601d 2024-11-12T12:26:40,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/0fc8d274354c4f0db14697889b71a941 is 50, key is test_row_0/C:col10/1731414398684/Put/seqid=0 2024-11-12T12:26:40,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742328_1504 (size=12301) 2024-11-12T12:26:40,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T12:26:40,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,908 INFO [Thread-1975 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1975. 2024-11-12T12:26:40,908 DEBUG [Thread-1975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:40,908 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,908 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:40,912 INFO [Thread-1973 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1973. 2024-11-12T12:26:40,912 DEBUG [Thread-1973 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:40,912 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,912 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:40,914 INFO [Thread-1971 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1971. 2024-11-12T12:26:40,914 DEBUG [Thread-1971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:40,914 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,914 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:40,917 INFO [Thread-1967 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1967. 2024-11-12T12:26:40,917 DEBUG [Thread-1967 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:40,917 DEBUG [Thread-1967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,917 DEBUG [Thread-1967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:40,922 INFO [Thread-1969 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1969. 2024-11-12T12:26:40,922 DEBUG [Thread-1969 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:40,922 DEBUG [Thread-1969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,922 DEBUG [Thread-1969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:40,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:40,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T12:26:40,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:40,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:40,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T12:26:41,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/0fc8d274354c4f0db14697889b71a941 2024-11-12T12:26:41,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/a7080f0f3f53445cbb0255de25431f5b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a7080f0f3f53445cbb0255de25431f5b 2024-11-12T12:26:41,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a7080f0f3f53445cbb0255de25431f5b, entries=200, sequenceid=332, filesize=14.4 K 2024-11-12T12:26:41,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/0cb7929e681e4a708a2a7788c699601d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/0cb7929e681e4a708a2a7788c699601d 2024-11-12T12:26:41,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/0cb7929e681e4a708a2a7788c699601d, entries=150, sequenceid=332, filesize=12.0 K 2024-11-12T12:26:41,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/0fc8d274354c4f0db14697889b71a941 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0fc8d274354c4f0db14697889b71a941 2024-11-12T12:26:41,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0fc8d274354c4f0db14697889b71a941, entries=150, sequenceid=332, filesize=12.0 K 2024-11-12T12:26:41,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c64ea0949ce45a18c8de4f8e776f777e in 1250ms, sequenceid=332, compaction requested=true 2024-11-12T12:26:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:41,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T12:26:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:41,094 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:41,094 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c64ea0949ce45a18c8de4f8e776f777e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:41,094 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-12T12:26:41,095 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:41,095 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:41,095 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/B is initiating minor compaction (all files) 2024-11-12T12:26:41,095 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/A is initiating minor compaction (all files) 2024-11-12T12:26:41,095 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/A in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:41,095 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/B in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:41,095 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/516739aa3d3b442a9c49d375abf80920, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/d2d23f3b902d46b092263adb5e7de7c7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/99d6511d12a44d459bc6f3286a2157f0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a7080f0f3f53445cbb0255de25431f5b] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=53.4 K 2024-11-12T12:26:41,095 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cfeadd534e1c4ee69ec784670a6b7617, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7c5869140eb84ba2b6ee52535e7217a0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/f3532c5c9fc4408f8c0d614264b0d027, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/0cb7929e681e4a708a2a7788c699601d] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=48.7 K 2024-11-12T12:26:41,096 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting cfeadd534e1c4ee69ec784670a6b7617, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731414395007 2024-11-12T12:26:41,096 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516739aa3d3b442a9c49d375abf80920, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731414395007 2024-11-12T12:26:41,096 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c5869140eb84ba2b6ee52535e7217a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731414396214 2024-11-12T12:26:41,096 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2d23f3b902d46b092263adb5e7de7c7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731414396191 2024-11-12T12:26:41,096 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting f3532c5c9fc4408f8c0d614264b0d027, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731414397500 2024-11-12T12:26:41,096 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d6511d12a44d459bc6f3286a2157f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731414397500 2024-11-12T12:26:41,096 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cb7929e681e4a708a2a7788c699601d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731414398671 2024-11-12T12:26:41,097 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7080f0f3f53445cbb0255de25431f5b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731414398671 2024-11-12T12:26:41,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/59d6af8a1ba4497aa88a00ce4a12a295 is 50, key is test_row_0/A:col10/1731414399985/Put/seqid=0 2024-11-12T12:26:41,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742329_1505 (size=12301) 2024-11-12T12:26:41,104 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#B#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:41,104 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#A#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:41,104 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/04905d063c554955b89bcc6bbc248beb is 50, key is test_row_0/A:col10/1731414398684/Put/seqid=0 2024-11-12T12:26:41,104 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/86a78700e8e14bce847a1b0e96c1989b is 50, key is test_row_0/B:col10/1731414398684/Put/seqid=0 2024-11-12T12:26:41,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742331_1507 (size=13085) 2024-11-12T12:26:41,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742330_1506 (size=13085) 2024-11-12T12:26:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:41,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. as already flushing 2024-11-12T12:26:41,154 INFO [Thread-1960 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1960. 2024-11-12T12:26:41,154 DEBUG [Thread-1960 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:41,154 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,154 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:41,503 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/59d6af8a1ba4497aa88a00ce4a12a295 2024-11-12T12:26:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/a056a25ac48e46c8966338b3affaded6 is 50, key is test_row_0/B:col10/1731414399985/Put/seqid=0 2024-11-12T12:26:41,511 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/86a78700e8e14bce847a1b0e96c1989b as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/86a78700e8e14bce847a1b0e96c1989b 2024-11-12T12:26:41,511 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/04905d063c554955b89bcc6bbc248beb as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/04905d063c554955b89bcc6bbc248beb 2024-11-12T12:26:41,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742332_1508 (size=12301) 2024-11-12T12:26:41,515 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/A of c64ea0949ce45a18c8de4f8e776f777e into 04905d063c554955b89bcc6bbc248beb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:41,515 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/B of c64ea0949ce45a18c8de4f8e776f777e into 86a78700e8e14bce847a1b0e96c1989b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:41,515 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:41,515 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:41,515 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/A, priority=12, startTime=1731414401094; duration=0sec 2024-11-12T12:26:41,515 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/B, priority=12, startTime=1731414401094; duration=0sec 2024-11-12T12:26:41,515 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:41,515 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:A 2024-11-12T12:26:41,515 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:41,515 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:B 2024-11-12T12:26:41,515 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T12:26:41,516 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T12:26:41,516 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): c64ea0949ce45a18c8de4f8e776f777e/C is initiating minor compaction (all files) 2024-11-12T12:26:41,516 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c64ea0949ce45a18c8de4f8e776f777e/C in TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:41,516 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62e68c35873b49afa4aeafc145d27670, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfa26dc4c33d49858ef2fa7e52d9ad39, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/761a4ea9aaed42ad8f434aa35f9fb623, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0fc8d274354c4f0db14697889b71a941] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp, totalSize=48.7 K 2024-11-12T12:26:41,517 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62e68c35873b49afa4aeafc145d27670, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731414395007 2024-11-12T12:26:41,517 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfa26dc4c33d49858ef2fa7e52d9ad39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731414396214 2024-11-12T12:26:41,517 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761a4ea9aaed42ad8f434aa35f9fb623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731414397500 2024-11-12T12:26:41,517 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc8d274354c4f0db14697889b71a941, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731414398671 2024-11-12T12:26:41,524 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c64ea0949ce45a18c8de4f8e776f777e#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:41,525 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/955c13cd197c4c028cb665d6881cd369 is 50, key is test_row_0/C:col10/1731414398684/Put/seqid=0 2024-11-12T12:26:41,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742333_1509 (size=13085) 2024-11-12T12:26:41,665 INFO [Thread-1956 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1956. 2024-11-12T12:26:41,665 DEBUG [Thread-1956 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:41,665 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,665 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:41,686 INFO [Thread-1964 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1964. 2024-11-12T12:26:41,686 INFO [Thread-1962 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1962. 2024-11-12T12:26:41,686 DEBUG [Thread-1964 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:41,686 DEBUG [Thread-1962 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:41,686 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,686 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,686 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,686 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:41,686 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,686 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:41,726 INFO [Thread-1958 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1958. 2024-11-12T12:26:41,726 DEBUG [Thread-1958 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:26:41,726 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,726 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:41,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:41,915 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/a056a25ac48e46c8966338b3affaded6 2024-11-12T12:26:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3c0947b435f84d9e8e29aad5c85c2f79 is 50, key is test_row_0/C:col10/1731414399985/Put/seqid=0 2024-11-12T12:26:41,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742334_1510 (size=12301) 2024-11-12T12:26:41,932 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/955c13cd197c4c028cb665d6881cd369 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/955c13cd197c4c028cb665d6881cd369 2024-11-12T12:26:41,936 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c64ea0949ce45a18c8de4f8e776f777e/C of c64ea0949ce45a18c8de4f8e776f777e into 955c13cd197c4c028cb665d6881cd369(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:41,936 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:41,936 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., storeName=c64ea0949ce45a18c8de4f8e776f777e/C, priority=12, startTime=1731414401094; duration=0sec 2024-11-12T12:26:41,937 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:41,937 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c64ea0949ce45a18c8de4f8e776f777e:C 2024-11-12T12:26:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T12:26:42,333 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3c0947b435f84d9e8e29aad5c85c2f79 2024-11-12T12:26:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/59d6af8a1ba4497aa88a00ce4a12a295 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/59d6af8a1ba4497aa88a00ce4a12a295 2024-11-12T12:26:42,407 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/59d6af8a1ba4497aa88a00ce4a12a295, entries=150, sequenceid=356, filesize=12.0 K 2024-11-12T12:26:42,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/a056a25ac48e46c8966338b3affaded6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/a056a25ac48e46c8966338b3affaded6 2024-11-12T12:26:42,410 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/a056a25ac48e46c8966338b3affaded6, entries=150, sequenceid=356, filesize=12.0 K 2024-11-12T12:26:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/3c0947b435f84d9e8e29aad5c85c2f79 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3c0947b435f84d9e8e29aad5c85c2f79 2024-11-12T12:26:42,414 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3c0947b435f84d9e8e29aad5c85c2f79, entries=150, sequenceid=356, filesize=12.0 K 2024-11-12T12:26:42,414 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=33.54 KB/34350 for c64ea0949ce45a18c8de4f8e776f777e in 1320ms, sequenceid=356, compaction requested=false 2024-11-12T12:26:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for c64ea0949ce45a18c8de4f8e776f777e: 2024-11-12T12:26:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-12T12:26:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-12T12:26:42,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-12T12:26:42,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5420 sec 2024-11-12T12:26:42,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.5470 sec 2024-11-12T12:26:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T12:26:44,052 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 231 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1432 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4294 rows 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1443 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4323 rows 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1434 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4302 rows 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1438 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4309 rows 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1435 2024-11-12T12:26:44,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4303 rows 2024-11-12T12:26:44,052 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T12:26:44,052 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T12:26:44,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:44,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:44,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:26:44,053 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T12:26:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T12:26:44,057 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414404056"}]},"ts":"1731414404056"} 2024-11-12T12:26:44,058 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T12:26:44,058 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T12:26:44,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T12:26:44,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c64ea0949ce45a18c8de4f8e776f777e, UNASSIGN}] 2024-11-12T12:26:44,061 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c64ea0949ce45a18c8de4f8e776f777e, UNASSIGN 2024-11-12T12:26:44,062 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c64ea0949ce45a18c8de4f8e776f777e, regionState=CLOSING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:44,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c64ea0949ce45a18c8de4f8e776f777e, UNASSIGN because future has completed 2024-11-12T12:26:44,064 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T12:26:44,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:26:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T12:26:44,216 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:44,216 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T12:26:44,216 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing c64ea0949ce45a18c8de4f8e776f777e, disabling compactions & flushes 2024-11-12T12:26:44,216 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:44,216 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:44,216 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. after waiting 0 ms 2024-11-12T12:26:44,216 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:44,216 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing c64ea0949ce45a18c8de4f8e776f777e 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-12T12:26:44,219 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/1f7d7a7e5b434e95abe56166a6d09e2f is 50, key is test_row_0/A:col10/1731414401723/Put/seqid=0 2024-11-12T12:26:44,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742335_1511 (size=9857) 2024-11-12T12:26:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T12:26:44,623 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/1f7d7a7e5b434e95abe56166a6d09e2f 2024-11-12T12:26:44,636 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/5101d6601e77462b878d8d02df00fb14 is 50, key is test_row_0/B:col10/1731414401723/Put/seqid=0 2024-11-12T12:26:44,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742336_1512 (size=9857) 2024-11-12T12:26:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T12:26:45,042 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/5101d6601e77462b878d8d02df00fb14 2024-11-12T12:26:45,056 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/59e9ae7f20074e0aa4fc710e73de3779 is 50, key is test_row_0/C:col10/1731414401723/Put/seqid=0 2024-11-12T12:26:45,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742337_1513 (size=9857) 2024-11-12T12:26:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T12:26:45,461 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/59e9ae7f20074e0aa4fc710e73de3779 2024-11-12T12:26:45,469 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/A/1f7d7a7e5b434e95abe56166a6d09e2f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/1f7d7a7e5b434e95abe56166a6d09e2f 2024-11-12T12:26:45,474 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/1f7d7a7e5b434e95abe56166a6d09e2f, entries=100, sequenceid=367, filesize=9.6 K 2024-11-12T12:26:45,475 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/B/5101d6601e77462b878d8d02df00fb14 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/5101d6601e77462b878d8d02df00fb14 2024-11-12T12:26:45,478 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/5101d6601e77462b878d8d02df00fb14, entries=100, sequenceid=367, filesize=9.6 K 2024-11-12T12:26:45,479 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/.tmp/C/59e9ae7f20074e0aa4fc710e73de3779 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/59e9ae7f20074e0aa4fc710e73de3779 2024-11-12T12:26:45,482 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/59e9ae7f20074e0aa4fc710e73de3779, entries=100, sequenceid=367, filesize=9.6 K 2024-11-12T12:26:45,482 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c64ea0949ce45a18c8de4f8e776f777e in 1266ms, sequenceid=367, compaction requested=true 2024-11-12T12:26:45,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/35f9bb034f50418aa137e3876c25e136, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/149348637b6b4d38abc926b55a9d6ee9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb81bbde2db84b28a5817c5522aceacb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/9ce836f730454d1f85e418878fd6e754, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/ab41e01b9bad4ac98ba4748b42cdaf4f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/f4cadacb5d804062821c598c5113f544, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/cd5997378ebd402a88e063daf2558755, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/466859199b394c33ba5c3a969d0deed0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/0293703c6cca49fe902a14452f668c83, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/5705c6f0304d4ef28ae3b7ed5d346fe6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/aea6833ebc8244999ac2394ecd40ea61, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/eea12c0eb3f44e85b23fd4f806b67cd3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb6efb82e53d40979e4a0379e8d6ae19, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a8307ca93d5e48c5a2be2a25bb215e16, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6f503409bdc24ed19976d8f27e1e8ff9, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/788c0a8d855d491f8f261831fae6dca0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/3f476690957e428ab84b4c8bfa774b58, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6c21a5af65b142d0905e48ffc04a5bbd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/c03152fd08054b34837b056d31789b29, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/516739aa3d3b442a9c49d375abf80920, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/d2d23f3b902d46b092263adb5e7de7c7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/99d6511d12a44d459bc6f3286a2157f0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a7080f0f3f53445cbb0255de25431f5b] to archive 2024-11-12T12:26:45,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:26:45,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/35f9bb034f50418aa137e3876c25e136 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/35f9bb034f50418aa137e3876c25e136 2024-11-12T12:26:45,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/149348637b6b4d38abc926b55a9d6ee9 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/149348637b6b4d38abc926b55a9d6ee9 2024-11-12T12:26:45,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb81bbde2db84b28a5817c5522aceacb to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb81bbde2db84b28a5817c5522aceacb 2024-11-12T12:26:45,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/9ce836f730454d1f85e418878fd6e754 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/9ce836f730454d1f85e418878fd6e754 2024-11-12T12:26:45,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/ab41e01b9bad4ac98ba4748b42cdaf4f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/ab41e01b9bad4ac98ba4748b42cdaf4f 2024-11-12T12:26:45,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/f4cadacb5d804062821c598c5113f544 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/f4cadacb5d804062821c598c5113f544 2024-11-12T12:26:45,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/cd5997378ebd402a88e063daf2558755 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/cd5997378ebd402a88e063daf2558755 2024-11-12T12:26:45,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/466859199b394c33ba5c3a969d0deed0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/466859199b394c33ba5c3a969d0deed0 2024-11-12T12:26:45,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/0293703c6cca49fe902a14452f668c83 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/0293703c6cca49fe902a14452f668c83 2024-11-12T12:26:45,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/5705c6f0304d4ef28ae3b7ed5d346fe6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/5705c6f0304d4ef28ae3b7ed5d346fe6 2024-11-12T12:26:45,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/aea6833ebc8244999ac2394ecd40ea61 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/aea6833ebc8244999ac2394ecd40ea61 2024-11-12T12:26:45,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/eea12c0eb3f44e85b23fd4f806b67cd3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/eea12c0eb3f44e85b23fd4f806b67cd3 2024-11-12T12:26:45,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb6efb82e53d40979e4a0379e8d6ae19 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/fb6efb82e53d40979e4a0379e8d6ae19 2024-11-12T12:26:45,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a8307ca93d5e48c5a2be2a25bb215e16 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a8307ca93d5e48c5a2be2a25bb215e16 2024-11-12T12:26:45,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6f503409bdc24ed19976d8f27e1e8ff9 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6f503409bdc24ed19976d8f27e1e8ff9 2024-11-12T12:26:45,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/788c0a8d855d491f8f261831fae6dca0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/788c0a8d855d491f8f261831fae6dca0 2024-11-12T12:26:45,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/3f476690957e428ab84b4c8bfa774b58 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/3f476690957e428ab84b4c8bfa774b58 2024-11-12T12:26:45,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6c21a5af65b142d0905e48ffc04a5bbd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/6c21a5af65b142d0905e48ffc04a5bbd 2024-11-12T12:26:45,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/c03152fd08054b34837b056d31789b29 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/c03152fd08054b34837b056d31789b29 2024-11-12T12:26:45,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/516739aa3d3b442a9c49d375abf80920 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/516739aa3d3b442a9c49d375abf80920 2024-11-12T12:26:45,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/d2d23f3b902d46b092263adb5e7de7c7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/d2d23f3b902d46b092263adb5e7de7c7 2024-11-12T12:26:45,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/99d6511d12a44d459bc6f3286a2157f0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/99d6511d12a44d459bc6f3286a2157f0 2024-11-12T12:26:45,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a7080f0f3f53445cbb0255de25431f5b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/a7080f0f3f53445cbb0255de25431f5b 2024-11-12T12:26:45,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b8c7cf3e5bb4400dbedfbfdc71e986ec, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/28ef4fd2db4e46b087a14a97386d4dc2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/5152fd61367e44bc86de4598cbbee017, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ae453f589f814397adfa755209adb8c1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cac977dc129c457eae4d495e61037a2d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/47d6f1c932274f288853e5327ce42d27, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/6b8f6ed3e9d5438f8b2280907ac05c8a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/04306de340cf48bda67b3bc598bed4be, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7b26eff3a29c49a69f1f7db048fb28cd, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/c4f6463383984e53b83a8b71ecb8ec21, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/354ad5b8d3cd419eb73bd6bffdf48083, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/987f9bbadd214d2281b855994b347cf3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fd70ee2243e34490bc0ea090d3d73e24, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b728cc835f264803b9c6e69662bc6139, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/3a9edcd61b1f41998ae9410ddb82ef82, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/408e39a23dcd41e498bd13df113f53ce, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fc251e67a1f44587805fcd920685434e, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/8e90f2c8f85442a6880e2e56cb5276f1, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cfeadd534e1c4ee69ec784670a6b7617, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ec12e20acf574de9a45b44761f864877, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7c5869140eb84ba2b6ee52535e7217a0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/f3532c5c9fc4408f8c0d614264b0d027, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/0cb7929e681e4a708a2a7788c699601d] to archive 2024-11-12T12:26:45,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:26:45,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b8c7cf3e5bb4400dbedfbfdc71e986ec to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b8c7cf3e5bb4400dbedfbfdc71e986ec 2024-11-12T12:26:45,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/28ef4fd2db4e46b087a14a97386d4dc2 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/28ef4fd2db4e46b087a14a97386d4dc2 2024-11-12T12:26:45,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/5152fd61367e44bc86de4598cbbee017 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/5152fd61367e44bc86de4598cbbee017 2024-11-12T12:26:45,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ae453f589f814397adfa755209adb8c1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ae453f589f814397adfa755209adb8c1 2024-11-12T12:26:45,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cac977dc129c457eae4d495e61037a2d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cac977dc129c457eae4d495e61037a2d 2024-11-12T12:26:45,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/47d6f1c932274f288853e5327ce42d27 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/47d6f1c932274f288853e5327ce42d27 2024-11-12T12:26:45,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/6b8f6ed3e9d5438f8b2280907ac05c8a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/6b8f6ed3e9d5438f8b2280907ac05c8a 2024-11-12T12:26:45,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/04306de340cf48bda67b3bc598bed4be to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/04306de340cf48bda67b3bc598bed4be 2024-11-12T12:26:45,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7b26eff3a29c49a69f1f7db048fb28cd to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7b26eff3a29c49a69f1f7db048fb28cd 2024-11-12T12:26:45,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/c4f6463383984e53b83a8b71ecb8ec21 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/c4f6463383984e53b83a8b71ecb8ec21 2024-11-12T12:26:45,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/354ad5b8d3cd419eb73bd6bffdf48083 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/354ad5b8d3cd419eb73bd6bffdf48083 2024-11-12T12:26:45,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/987f9bbadd214d2281b855994b347cf3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/987f9bbadd214d2281b855994b347cf3 2024-11-12T12:26:45,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fd70ee2243e34490bc0ea090d3d73e24 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fd70ee2243e34490bc0ea090d3d73e24 2024-11-12T12:26:45,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b728cc835f264803b9c6e69662bc6139 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/b728cc835f264803b9c6e69662bc6139 2024-11-12T12:26:45,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/3a9edcd61b1f41998ae9410ddb82ef82 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/3a9edcd61b1f41998ae9410ddb82ef82 2024-11-12T12:26:45,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/408e39a23dcd41e498bd13df113f53ce to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/408e39a23dcd41e498bd13df113f53ce 2024-11-12T12:26:45,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fc251e67a1f44587805fcd920685434e to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/fc251e67a1f44587805fcd920685434e 2024-11-12T12:26:45,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/8e90f2c8f85442a6880e2e56cb5276f1 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/8e90f2c8f85442a6880e2e56cb5276f1 2024-11-12T12:26:45,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cfeadd534e1c4ee69ec784670a6b7617 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/cfeadd534e1c4ee69ec784670a6b7617 2024-11-12T12:26:45,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ec12e20acf574de9a45b44761f864877 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/ec12e20acf574de9a45b44761f864877 2024-11-12T12:26:45,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7c5869140eb84ba2b6ee52535e7217a0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/7c5869140eb84ba2b6ee52535e7217a0 2024-11-12T12:26:45,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/f3532c5c9fc4408f8c0d614264b0d027 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/f3532c5c9fc4408f8c0d614264b0d027 2024-11-12T12:26:45,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/0cb7929e681e4a708a2a7788c699601d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/0cb7929e681e4a708a2a7788c699601d 2024-11-12T12:26:45,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9443573e882a406f9c1e9cbb69eb2b52, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/c794ae16c37b4eaf96c8cd1d2b224184, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/cd006a1ccbed4cd0aad5e7f26eb4e710, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9194c5f646954bff96372e8f51573502, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/448348853eb14fdebdfb46de19fbe2b3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfb0923db70643ebac1b1c729c5ca948, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3de0b191b5d2419e904c7a04b3822e8c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3b96956b2d124b5b830472d0b83aa6e0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a28f3358b7240658f8355d88597cc44, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/b35d17991bc549e88806c24b7706e16a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/eab902f74d4f4f6681a2af38ff60f272, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62f136ac2162471d985c56897d57b86a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/66919bf4347543ecb38bd35a00ccfd5d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/6db272dc66de47658e52b01779aecd29, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/1f4b43cd551c403396e9bd16ba42f86b, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/ed52e08db1df4ae59b6ee9100fd4bab2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3faf53c8cd0a43429967518d76633fe3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a7bb94f9a2f436682e8f54d0edb96e7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62e68c35873b49afa4aeafc145d27670, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0aa24b7b84d0486c9104a0c701bfd360, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfa26dc4c33d49858ef2fa7e52d9ad39, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/761a4ea9aaed42ad8f434aa35f9fb623, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0fc8d274354c4f0db14697889b71a941] to archive 2024-11-12T12:26:45,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:26:45,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9443573e882a406f9c1e9cbb69eb2b52 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9443573e882a406f9c1e9cbb69eb2b52 2024-11-12T12:26:45,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/c794ae16c37b4eaf96c8cd1d2b224184 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/c794ae16c37b4eaf96c8cd1d2b224184 2024-11-12T12:26:45,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/cd006a1ccbed4cd0aad5e7f26eb4e710 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/cd006a1ccbed4cd0aad5e7f26eb4e710 2024-11-12T12:26:45,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9194c5f646954bff96372e8f51573502 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/9194c5f646954bff96372e8f51573502 2024-11-12T12:26:45,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/448348853eb14fdebdfb46de19fbe2b3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/448348853eb14fdebdfb46de19fbe2b3 2024-11-12T12:26:45,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfb0923db70643ebac1b1c729c5ca948 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfb0923db70643ebac1b1c729c5ca948 2024-11-12T12:26:45,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3de0b191b5d2419e904c7a04b3822e8c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3de0b191b5d2419e904c7a04b3822e8c 2024-11-12T12:26:45,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3b96956b2d124b5b830472d0b83aa6e0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3b96956b2d124b5b830472d0b83aa6e0 2024-11-12T12:26:45,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a28f3358b7240658f8355d88597cc44 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a28f3358b7240658f8355d88597cc44 2024-11-12T12:26:45,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/b35d17991bc549e88806c24b7706e16a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/b35d17991bc549e88806c24b7706e16a 2024-11-12T12:26:45,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/eab902f74d4f4f6681a2af38ff60f272 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/eab902f74d4f4f6681a2af38ff60f272 2024-11-12T12:26:45,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62f136ac2162471d985c56897d57b86a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62f136ac2162471d985c56897d57b86a 2024-11-12T12:26:45,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/66919bf4347543ecb38bd35a00ccfd5d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/66919bf4347543ecb38bd35a00ccfd5d 2024-11-12T12:26:45,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/6db272dc66de47658e52b01779aecd29 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/6db272dc66de47658e52b01779aecd29 2024-11-12T12:26:45,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/1f4b43cd551c403396e9bd16ba42f86b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/1f4b43cd551c403396e9bd16ba42f86b 2024-11-12T12:26:45,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/ed52e08db1df4ae59b6ee9100fd4bab2 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/ed52e08db1df4ae59b6ee9100fd4bab2 2024-11-12T12:26:45,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3faf53c8cd0a43429967518d76633fe3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3faf53c8cd0a43429967518d76633fe3 2024-11-12T12:26:45,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a7bb94f9a2f436682e8f54d0edb96e7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/8a7bb94f9a2f436682e8f54d0edb96e7 2024-11-12T12:26:45,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62e68c35873b49afa4aeafc145d27670 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/62e68c35873b49afa4aeafc145d27670 2024-11-12T12:26:45,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0aa24b7b84d0486c9104a0c701bfd360 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0aa24b7b84d0486c9104a0c701bfd360 2024-11-12T12:26:45,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfa26dc4c33d49858ef2fa7e52d9ad39 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/dfa26dc4c33d49858ef2fa7e52d9ad39 2024-11-12T12:26:45,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/761a4ea9aaed42ad8f434aa35f9fb623 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/761a4ea9aaed42ad8f434aa35f9fb623 2024-11-12T12:26:45,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0fc8d274354c4f0db14697889b71a941 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/0fc8d274354c4f0db14697889b71a941 2024-11-12T12:26:45,552 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/recovered.edits/370.seqid, newMaxSeqId=370, maxSeqId=1 2024-11-12T12:26:45,552 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. 2024-11-12T12:26:45,553 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for c64ea0949ce45a18c8de4f8e776f777e: Waiting for close lock at 1731414404216Running coprocessor pre-close hooks at 1731414404216Disabling compacts and flushes for region at 1731414404216Disabling writes for close at 1731414404216Obtaining lock to block concurrent updates at 1731414404216Preparing flush snapshotting stores in c64ea0949ce45a18c8de4f8e776f777e at 1731414404216Finished memstore snapshotting TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731414404216Flushing stores of TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e. at 1731414404217 (+1 ms)Flushing c64ea0949ce45a18c8de4f8e776f777e/A: creating writer at 1731414404217Flushing c64ea0949ce45a18c8de4f8e776f777e/A: appending metadata at 1731414404219 (+2 ms)Flushing c64ea0949ce45a18c8de4f8e776f777e/A: closing flushed file at 1731414404219Flushing c64ea0949ce45a18c8de4f8e776f777e/B: creating writer at 1731414404631 (+412 ms)Flushing c64ea0949ce45a18c8de4f8e776f777e/B: appending metadata at 1731414404636 (+5 ms)Flushing c64ea0949ce45a18c8de4f8e776f777e/B: closing flushed file at 1731414404636Flushing c64ea0949ce45a18c8de4f8e776f777e/C: creating writer at 1731414405052 (+416 ms)Flushing c64ea0949ce45a18c8de4f8e776f777e/C: appending metadata at 1731414405055 (+3 ms)Flushing c64ea0949ce45a18c8de4f8e776f777e/C: closing flushed file at 1731414405055Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dcfe2f8: reopening flushed file at 1731414405468 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6140a113: reopening flushed file at 1731414405474 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3be5248a: reopening flushed file at 1731414405478 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c64ea0949ce45a18c8de4f8e776f777e in 1266ms, sequenceid=367, compaction requested=true at 1731414405482 (+4 ms)Writing region close event to WAL at 1731414405548 (+66 ms)Running coprocessor post-close hooks at 1731414405552 (+4 ms)Closed at 1731414405552 2024-11-12T12:26:45,555 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:45,556 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c64ea0949ce45a18c8de4f8e776f777e, regionState=CLOSED 2024-11-12T12:26:45,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:26:45,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-12T12:26:45,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure c64ea0949ce45a18c8de4f8e776f777e, server=4a91c05c96a4,38717,1731414257691 in 1.4930 sec 2024-11-12T12:26:45,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-12T12:26:45,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c64ea0949ce45a18c8de4f8e776f777e, UNASSIGN in 1.4990 sec 2024-11-12T12:26:45,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-12T12:26:45,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5020 sec 2024-11-12T12:26:45,563 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414405563"}]},"ts":"1731414405563"} 2024-11-12T12:26:45,565 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T12:26:45,565 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T12:26:45,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5130 sec 2024-11-12T12:26:46,047 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T12:26:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T12:26:46,203 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:46,205 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T12:26:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:46,209 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T12:26:46,211 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:46,215 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:46,218 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/recovered.edits] 2024-11-12T12:26:46,222 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/04905d063c554955b89bcc6bbc248beb to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/04905d063c554955b89bcc6bbc248beb 2024-11-12T12:26:46,225 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/1f7d7a7e5b434e95abe56166a6d09e2f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/1f7d7a7e5b434e95abe56166a6d09e2f 2024-11-12T12:26:46,227 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/59d6af8a1ba4497aa88a00ce4a12a295 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/A/59d6af8a1ba4497aa88a00ce4a12a295 2024-11-12T12:26:46,231 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/5101d6601e77462b878d8d02df00fb14 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/5101d6601e77462b878d8d02df00fb14 2024-11-12T12:26:46,233 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/86a78700e8e14bce847a1b0e96c1989b to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/86a78700e8e14bce847a1b0e96c1989b 2024-11-12T12:26:46,235 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/a056a25ac48e46c8966338b3affaded6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/B/a056a25ac48e46c8966338b3affaded6 2024-11-12T12:26:46,239 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3c0947b435f84d9e8e29aad5c85c2f79 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/3c0947b435f84d9e8e29aad5c85c2f79 2024-11-12T12:26:46,240 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/59e9ae7f20074e0aa4fc710e73de3779 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/59e9ae7f20074e0aa4fc710e73de3779 2024-11-12T12:26:46,241 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/955c13cd197c4c028cb665d6881cd369 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/C/955c13cd197c4c028cb665d6881cd369 2024-11-12T12:26:46,243 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/recovered.edits/370.seqid to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e/recovered.edits/370.seqid 2024-11-12T12:26:46,244 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/c64ea0949ce45a18c8de4f8e776f777e 2024-11-12T12:26:46,244 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T12:26:46,245 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:46,247 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T12:26:46,249 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T12:26:46,250 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:46,250 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T12:26:46,250 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731414406250"}]},"ts":"9223372036854775807"} 2024-11-12T12:26:46,251 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T12:26:46,251 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c64ea0949ce45a18c8de4f8e776f777e, NAME => 'TestAcidGuarantees,,1731414378707.c64ea0949ce45a18c8de4f8e776f777e.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T12:26:46,251 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T12:26:46,252 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731414406251"}]},"ts":"9223372036854775807"} 2024-11-12T12:26:46,253 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T12:26:46,253 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:46,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-12T12:26:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T12:26:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T12:26:46,322 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:46,332 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 237) - Thread LEAK? -, OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 520), ProcessCount=11 (was 11), AvailableMemoryMB=6712 (was 6734) 2024-11-12T12:26:46,342 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=476, ProcessCount=11, AvailableMemoryMB=6712 2024-11-12T12:26:46,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T12:26:46,344 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:26:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:46,346 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T12:26:46,346 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:46,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-12T12:26:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T12:26:46,347 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T12:26:46,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742338_1514 (size=959) 2024-11-12T12:26:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T12:26:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T12:26:46,759 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a 2024-11-12T12:26:46,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742339_1515 (size=53) 2024-11-12T12:26:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T12:26:47,171 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:26:47,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing de303568766865fcf42f0cc60107a910, disabling compactions & flushes 2024-11-12T12:26:47,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:47,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:47,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. after waiting 0 ms 2024-11-12T12:26:47,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:47,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:47,172 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for de303568766865fcf42f0cc60107a910: Waiting for close lock at 1731414407172Disabling compacts and flushes for region at 1731414407172Disabling writes for close at 1731414407172Writing region close event to WAL at 1731414407172Closed at 1731414407172 2024-11-12T12:26:47,175 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T12:26:47,176 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731414407175"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731414407175"}]},"ts":"1731414407175"} 2024-11-12T12:26:47,178 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T12:26:47,179 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T12:26:47,180 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414407179"}]},"ts":"1731414407179"} 2024-11-12T12:26:47,182 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T12:26:47,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, ASSIGN}] 2024-11-12T12:26:47,183 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, ASSIGN 2024-11-12T12:26:47,184 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, ASSIGN; state=OFFLINE, location=4a91c05c96a4,38717,1731414257691; forceNewPlan=false, retain=false 2024-11-12T12:26:47,286 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T12:26:47,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=de303568766865fcf42f0cc60107a910, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:47,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, ASSIGN because future has completed 2024-11-12T12:26:47,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:26:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T12:26:47,501 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:47,502 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:26:47,502 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,503 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:26:47,503 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,503 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,505 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,508 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de303568766865fcf42f0cc60107a910 columnFamilyName A 2024-11-12T12:26:47,508 DEBUG [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:47,509 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(327): Store=de303568766865fcf42f0cc60107a910/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:26:47,509 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,511 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de303568766865fcf42f0cc60107a910 columnFamilyName B 2024-11-12T12:26:47,511 DEBUG [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:47,512 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(327): Store=de303568766865fcf42f0cc60107a910/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:26:47,512 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,513 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de303568766865fcf42f0cc60107a910 columnFamilyName C 2024-11-12T12:26:47,513 DEBUG [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:47,514 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(327): Store=de303568766865fcf42f0cc60107a910/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:26:47,514 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,515 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,515 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,516 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,516 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,517 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:26:47,518 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,520 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T12:26:47,520 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened de303568766865fcf42f0cc60107a910; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62482045, jitterRate=-0.06894497573375702}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:26:47,520 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:47,521 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for de303568766865fcf42f0cc60107a910: Running coprocessor pre-open hook at 1731414407503Writing region info on filesystem at 1731414407503Initializing all the Stores at 1731414407504 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414407504Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414407505 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414407505Cleaning up temporary data from old regions at 1731414407516 (+11 ms)Running coprocessor post-open hooks at 1731414407520 (+4 ms)Region opened successfully at 1731414407521 (+1 ms) 2024-11-12T12:26:47,522 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., pid=142, masterSystemTime=1731414407494 2024-11-12T12:26:47,523 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:47,524 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:47,524 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=de303568766865fcf42f0cc60107a910, regionState=OPEN, openSeqNum=2, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:47,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:26:47,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-12T12:26:47,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 in 186 msec 2024-11-12T12:26:47,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-12T12:26:47,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, ASSIGN in 345 msec 2024-11-12T12:26:47,530 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T12:26:47,530 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414407530"}]},"ts":"1731414407530"} 2024-11-12T12:26:47,531 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T12:26:47,532 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T12:26:47,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-12T12:26:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T12:26:48,493 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:48,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:48,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:48,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:48,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e07d6f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:48,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:48,503 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48bcc92d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:48,504 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:48,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:48,506 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T12:26:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 4a91c05c96a4,41939,1731414256942 2024-11-12T12:26:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f40357c 2024-11-12T12:26:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T12:26:48,508 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T12:26:48,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T12:26:48,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T12:26:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T12:26:48,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742340_1516 (size=995) 2024-11-12T12:26:48,922 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-12T12:26:48,922 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-12T12:26:48,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T12:26:48,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, REOPEN/MOVE}] 2024-11-12T12:26:48,932 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, REOPEN/MOVE 2024-11-12T12:26:48,933 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=de303568766865fcf42f0cc60107a910, regionState=CLOSING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:48,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, REOPEN/MOVE because future has completed 2024-11-12T12:26:48,936 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T12:26:48,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:26:49,090 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,090 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T12:26:49,090 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing de303568766865fcf42f0cc60107a910, disabling compactions & flushes 2024-11-12T12:26:49,090 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,090 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,090 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. after waiting 0 ms 2024-11-12T12:26:49,090 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,099 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T12:26:49,101 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,101 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for de303568766865fcf42f0cc60107a910: Waiting for close lock at 1731414409090Running coprocessor pre-close hooks at 1731414409090Disabling compacts and flushes for region at 1731414409090Disabling writes for close at 1731414409090Writing region close event to WAL at 1731414409091 (+1 ms)Running coprocessor post-close hooks at 1731414409100 (+9 ms)Closed at 1731414409100 2024-11-12T12:26:49,101 WARN [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: de303568766865fcf42f0cc60107a910 to self. 2024-11-12T12:26:49,103 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,104 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=de303568766865fcf42f0cc60107a910, regionState=CLOSED 2024-11-12T12:26:49,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:26:49,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-12T12:26:49,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 in 171 msec 2024-11-12T12:26:49,110 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, REOPEN/MOVE; state=CLOSED, location=4a91c05c96a4,38717,1731414257691; forceNewPlan=false, retain=true 2024-11-12T12:26:49,261 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=de303568766865fcf42f0cc60107a910, regionState=OPENING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, REOPEN/MOVE because future has completed 2024-11-12T12:26:49,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:26:49,419 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,419 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} 2024-11-12T12:26:49,419 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,420 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T12:26:49,420 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,420 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,422 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,424 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de303568766865fcf42f0cc60107a910 columnFamilyName A 2024-11-12T12:26:49,425 DEBUG [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:49,426 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(327): Store=de303568766865fcf42f0cc60107a910/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:26:49,426 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,427 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de303568766865fcf42f0cc60107a910 columnFamilyName B 2024-11-12T12:26:49,427 DEBUG [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:49,428 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(327): Store=de303568766865fcf42f0cc60107a910/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:26:49,428 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,428 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de303568766865fcf42f0cc60107a910 columnFamilyName C 2024-11-12T12:26:49,428 DEBUG [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:49,428 INFO [StoreOpener-de303568766865fcf42f0cc60107a910-1 {}] regionserver.HStore(327): Store=de303568766865fcf42f0cc60107a910/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T12:26:49,429 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,429 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,430 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,430 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,430 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,431 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T12:26:49,431 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,432 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened de303568766865fcf42f0cc60107a910; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62625158, jitterRate=-0.0668124258518219}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T12:26:49,432 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,432 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for de303568766865fcf42f0cc60107a910: Running coprocessor pre-open hook at 1731414409420Writing region info on filesystem at 1731414409420Initializing all the Stores at 1731414409421 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414409422 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414409422Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731414409422Cleaning up temporary data from old regions at 1731414409430 (+8 ms)Running coprocessor post-open hooks at 1731414409432 (+2 ms)Region opened successfully at 1731414409432 2024-11-12T12:26:49,433 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., pid=147, masterSystemTime=1731414409414 2024-11-12T12:26:49,434 DEBUG [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,434 INFO [RS_OPEN_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,435 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=de303568766865fcf42f0cc60107a910, regionState=OPEN, openSeqNum=5, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:26:49,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-12T12:26:49,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 in 174 msec 2024-11-12T12:26:49,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-12T12:26:49,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, REOPEN/MOVE in 507 msec 2024-11-12T12:26:49,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-12T12:26:49,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-11-12T12:26:49,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 932 msec 2024-11-12T12:26:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-12T12:26:49,443 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:49,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,445 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca3c99c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ea6d7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,448 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738d6326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,449 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@778266d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,451 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2242432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,452 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd2e75e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,453 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4423f9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,454 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604b35b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,456 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738d8fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,457 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b166bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,458 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,459 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e84546b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,460 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159b3e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,461 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70811740, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,462 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dfebfab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,463 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c381193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,464 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40681b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,466 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4de9d449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,467 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69856d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4a91c05c96a4,41939,-1 for getting cluster id 2024-11-12T12:26:49,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T12:26:49,468 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a16b6acc-0c60-49b2-bc4a-28b799cceb49' 2024-11-12T12:26:49,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T12:26:49,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a16b6acc-0c60-49b2-bc4a-28b799cceb49" 2024-11-12T12:26:49,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7feb05af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4a91c05c96a4,41939,-1] 2024-11-12T12:26:49,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T12:26:49,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:26:49,469 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T12:26:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22533c98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T12:26:49,470 DEBUG [Thread-2329 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,470 DEBUG [Thread-2323 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,470 DEBUG [Thread-2327 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,470 DEBUG [Thread-2321 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,471 DEBUG [Thread-2312 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,471 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,471 DEBUG [Thread-2318 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,471 DEBUG [Thread-2314 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,471 DEBUG [Thread-2316 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,472 DEBUG [Thread-2325 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,472 DEBUG [Thread-2310 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T12:26:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-12T12:26:49,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,473 DEBUG [Thread-2325 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,473 DEBUG [Thread-2325 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,474 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T12:26:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4a91c05c96a4,38717,1731414257691, seqNum=-1] 2024-11-12T12:26:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T12:26:49,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,475 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:49,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:49,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T12:26:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5] 2024-11-12T12:26:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,482 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:49,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:26:49,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414469503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414469503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414469504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d57025c44fc04537b1a03ed420dbb5d5_de303568766865fcf42f0cc60107a910 is 50, key is test_row_1/A:col10/1731414409480/Put/seqid=0 2024-11-12T12:26:49,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414469506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414469506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742341_1517 (size=12154) 2024-11-12T12:26:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T12:26:49,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414469611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414469611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414469611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414469611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414469611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T12:26:49,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:49,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:49,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:49,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T12:26:49,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:49,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:49,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T12:26:49,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414469821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414469821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414469821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414469821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414469821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:49,918 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:49,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d57025c44fc04537b1a03ed420dbb5d5_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d57025c44fc04537b1a03ed420dbb5d5_de303568766865fcf42f0cc60107a910 2024-11-12T12:26:49,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/3ea8af1d8a874a6a869bc9dd85761d1a, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:49,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/3ea8af1d8a874a6a869bc9dd85761d1a is 175, key is test_row_1/A:col10/1731414409480/Put/seqid=0 2024-11-12T12:26:49,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742342_1518 (size=30951) 2024-11-12T12:26:49,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/3ea8af1d8a874a6a869bc9dd85761d1a 2024-11-12T12:26:49,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T12:26:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:49,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:49,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/e420809df4814c069750bace5418edcc is 50, key is test_row_1/B:col10/1731414409480/Put/seqid=0 2024-11-12T12:26:49,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742343_1519 (size=9657) 2024-11-12T12:26:50,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T12:26:50,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:50,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:50,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:50,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:50,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T12:26:50,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414470131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414470130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414470131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414470131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414470131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T12:26:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:50,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:50,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/e420809df4814c069750bace5418edcc 2024-11-12T12:26:50,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/b88c7deb244f4f16a4c1a79e412c04ac is 50, key is test_row_1/C:col10/1731414409480/Put/seqid=0 2024-11-12T12:26:50,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742344_1520 (size=9657) 2024-11-12T12:26:50,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/b88c7deb244f4f16a4c1a79e412c04ac 2024-11-12T12:26:50,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/3ea8af1d8a874a6a869bc9dd85761d1a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/3ea8af1d8a874a6a869bc9dd85761d1a 2024-11-12T12:26:50,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/3ea8af1d8a874a6a869bc9dd85761d1a, entries=150, sequenceid=15, filesize=30.2 K 2024-11-12T12:26:50,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/e420809df4814c069750bace5418edcc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/e420809df4814c069750bace5418edcc 2024-11-12T12:26:50,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/e420809df4814c069750bace5418edcc, entries=100, sequenceid=15, filesize=9.4 K 2024-11-12T12:26:50,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/b88c7deb244f4f16a4c1a79e412c04ac as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b88c7deb244f4f16a4c1a79e412c04ac 2024-11-12T12:26:50,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b88c7deb244f4f16a4c1a79e412c04ac, entries=100, sequenceid=15, filesize=9.4 K 2024-11-12T12:26:50,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for de303568766865fcf42f0cc60107a910 in 909ms, sequenceid=15, compaction requested=false 2024-11-12T12:26:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:50,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T12:26:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:50,392 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-12T12:26:50,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121e2d246a17c34f8ca3b95210434f2c87_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414409503/Put/seqid=0 2024-11-12T12:26:50,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742345_1521 (size=12154) 2024-11-12T12:26:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T12:26:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:26:50,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:50,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414470644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414470645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414470645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414470646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414470648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414470751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414470751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414470751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414470751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414470760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:50,810 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121e2d246a17c34f8ca3b95210434f2c87_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121e2d246a17c34f8ca3b95210434f2c87_de303568766865fcf42f0cc60107a910 2024-11-12T12:26:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/466f9beceb34492ebad4bb3f3446f8f5, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:50,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/466f9beceb34492ebad4bb3f3446f8f5 is 175, key is test_row_0/A:col10/1731414409503/Put/seqid=0 2024-11-12T12:26:50,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742346_1522 (size=30955) 2024-11-12T12:26:50,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414470961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414470961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414470961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414470961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:50,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414470970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,074 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T12:26:51,232 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/466f9beceb34492ebad4bb3f3446f8f5 2024-11-12T12:26:51,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/9d4000935c4e4fe6b8cfab1abcc01301 is 50, key is test_row_0/B:col10/1731414409503/Put/seqid=0 2024-11-12T12:26:51,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742347_1523 (size=12001) 2024-11-12T12:26:51,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414471271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414471271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414471271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414471271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414471281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T12:26:51,641 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/9d4000935c4e4fe6b8cfab1abcc01301 2024-11-12T12:26:51,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/dae957eef59d43b6971e3ba09ad1c8ab is 50, key is test_row_0/C:col10/1731414409503/Put/seqid=0 2024-11-12T12:26:51,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742348_1524 (size=12001) 2024-11-12T12:26:51,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414471781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414471781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414471781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414471781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:51,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414471791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,062 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/dae957eef59d43b6971e3ba09ad1c8ab 2024-11-12T12:26:52,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/466f9beceb34492ebad4bb3f3446f8f5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/466f9beceb34492ebad4bb3f3446f8f5 2024-11-12T12:26:52,068 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/466f9beceb34492ebad4bb3f3446f8f5, entries=150, sequenceid=41, filesize=30.2 K 2024-11-12T12:26:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/9d4000935c4e4fe6b8cfab1abcc01301 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/9d4000935c4e4fe6b8cfab1abcc01301 2024-11-12T12:26:52,070 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/9d4000935c4e4fe6b8cfab1abcc01301, entries=150, sequenceid=41, filesize=11.7 K 2024-11-12T12:26:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/dae957eef59d43b6971e3ba09ad1c8ab as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/dae957eef59d43b6971e3ba09ad1c8ab 2024-11-12T12:26:52,073 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/dae957eef59d43b6971e3ba09ad1c8ab, entries=150, sequenceid=41, filesize=11.7 K 2024-11-12T12:26:52,074 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for de303568766865fcf42f0cc60107a910 in 1682ms, sequenceid=41, compaction requested=false 2024-11-12T12:26:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-12T12:26:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-12T12:26:52,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-12T12:26:52,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6000 sec 2024-11-12T12:26:52,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.6060 sec 2024-11-12T12:26:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:26:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:52,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-12T12:26:52,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125c10c4d66c18411ea5fddf66fcf53673_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414410644/Put/seqid=0 2024-11-12T12:26:52,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742349_1525 (size=12154) 2024-11-12T12:26:52,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414472814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414472814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414472818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414472819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414472819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414472921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414472931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414472931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414472931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414472931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414473131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414473141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414473141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414473141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414473141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,206 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:53,208 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125c10c4d66c18411ea5fddf66fcf53673_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125c10c4d66c18411ea5fddf66fcf53673_de303568766865fcf42f0cc60107a910 2024-11-12T12:26:53,209 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/229554eba65c4a6888418b6594105972, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:53,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/229554eba65c4a6888418b6594105972 is 175, key is test_row_0/A:col10/1731414410644/Put/seqid=0 2024-11-12T12:26:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742350_1526 (size=30955) 2024-11-12T12:26:53,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414473441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414473451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414473451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414473451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414473451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,612 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/229554eba65c4a6888418b6594105972 2024-11-12T12:26:53,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/095a02c8359240599d56f27014e345f7 is 50, key is test_row_0/B:col10/1731414410644/Put/seqid=0 2024-11-12T12:26:53,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742351_1527 (size=12001) 2024-11-12T12:26:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T12:26:53,642 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:53,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-12T12:26:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T12:26:53,643 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:53,644 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:53,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T12:26:53,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T12:26:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:53,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:53,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T12:26:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:53,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:53,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414473951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T12:26:53,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414473962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414473962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414473962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414473962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:54,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/095a02c8359240599d56f27014e345f7 2024-11-12T12:26:54,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/1cdf6df69bee401a93c1d58d44b488a6 is 50, key is test_row_0/C:col10/1731414410644/Put/seqid=0 2024-11-12T12:26:54,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742352_1528 (size=12001) 2024-11-12T12:26:54,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T12:26:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:54,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T12:26:54,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:54,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:54,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T12:26:54,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T12:26:54,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:54,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:54,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:54,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/1cdf6df69bee401a93c1d58d44b488a6 2024-11-12T12:26:54,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/229554eba65c4a6888418b6594105972 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/229554eba65c4a6888418b6594105972 2024-11-12T12:26:54,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/229554eba65c4a6888418b6594105972, entries=150, sequenceid=52, filesize=30.2 K 2024-11-12T12:26:54,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/095a02c8359240599d56f27014e345f7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/095a02c8359240599d56f27014e345f7 2024-11-12T12:26:54,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/095a02c8359240599d56f27014e345f7, entries=150, sequenceid=52, filesize=11.7 K 2024-11-12T12:26:54,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/1cdf6df69bee401a93c1d58d44b488a6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/1cdf6df69bee401a93c1d58d44b488a6 2024-11-12T12:26:54,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/1cdf6df69bee401a93c1d58d44b488a6, entries=150, sequenceid=52, filesize=11.7 K 2024-11-12T12:26:54,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for de303568766865fcf42f0cc60107a910 in 1663ms, sequenceid=52, compaction requested=true 2024-11-12T12:26:54,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:54,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:54,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:54,456 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:54,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:54,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:54,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:54,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T12:26:54,456 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:54,457 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:54,457 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:54,457 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/B is initiating minor compaction (all files) 2024-11-12T12:26:54,457 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/A is initiating minor compaction (all files) 2024-11-12T12:26:54,457 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/A in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,457 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/B in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,457 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/e420809df4814c069750bace5418edcc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/9d4000935c4e4fe6b8cfab1abcc01301, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/095a02c8359240599d56f27014e345f7] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=32.9 K 2024-11-12T12:26:54,457 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/3ea8af1d8a874a6a869bc9dd85761d1a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/466f9beceb34492ebad4bb3f3446f8f5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/229554eba65c4a6888418b6594105972] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=90.7 K 2024-11-12T12:26:54,457 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,457 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/3ea8af1d8a874a6a869bc9dd85761d1a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/466f9beceb34492ebad4bb3f3446f8f5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/229554eba65c4a6888418b6594105972] 2024-11-12T12:26:54,458 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ea8af1d8a874a6a869bc9dd85761d1a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731414409480 2024-11-12T12:26:54,458 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting e420809df4814c069750bace5418edcc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731414409480 2024-11-12T12:26:54,458 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466f9beceb34492ebad4bb3f3446f8f5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731414409502 2024-11-12T12:26:54,458 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4000935c4e4fe6b8cfab1abcc01301, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731414409502 2024-11-12T12:26:54,458 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 229554eba65c4a6888418b6594105972, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414410644 2024-11-12T12:26:54,458 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 095a02c8359240599d56f27014e345f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414410644 2024-11-12T12:26:54,464 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:54,469 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411120cc8e351e3bc467691cdf337c4d1575a_de303568766865fcf42f0cc60107a910 store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:54,470 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:54,470 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/6984b1eb67d849ed8be0582d895da3b7 is 50, key is test_row_0/B:col10/1731414410644/Put/seqid=0 2024-11-12T12:26:54,471 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411120cc8e351e3bc467691cdf337c4d1575a_de303568766865fcf42f0cc60107a910, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:54,471 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120cc8e351e3bc467691cdf337c4d1575a_de303568766865fcf42f0cc60107a910 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:54,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742353_1529 (size=4469) 2024-11-12T12:26:54,483 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#A#compaction#447 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:54,484 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/f33c3fe469774293849b335e9cef2006 is 175, key is test_row_0/A:col10/1731414410644/Put/seqid=0 2024-11-12T12:26:54,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742354_1530 (size=12104) 2024-11-12T12:26:54,495 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/6984b1eb67d849ed8be0582d895da3b7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6984b1eb67d849ed8be0582d895da3b7 2024-11-12T12:26:54,499 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/B of de303568766865fcf42f0cc60107a910 into 6984b1eb67d849ed8be0582d895da3b7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:54,499 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:54,500 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/B, priority=13, startTime=1731414414456; duration=0sec 2024-11-12T12:26:54,500 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:54,500 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:B 2024-11-12T12:26:54,500 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:54,501 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:54,501 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/C is initiating minor compaction (all files) 2024-11-12T12:26:54,501 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/C in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,501 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b88c7deb244f4f16a4c1a79e412c04ac, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/dae957eef59d43b6971e3ba09ad1c8ab, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/1cdf6df69bee401a93c1d58d44b488a6] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=32.9 K 2024-11-12T12:26:54,501 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting b88c7deb244f4f16a4c1a79e412c04ac, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731414409480 2024-11-12T12:26:54,502 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting dae957eef59d43b6971e3ba09ad1c8ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731414409502 2024-11-12T12:26:54,502 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cdf6df69bee401a93c1d58d44b488a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414410644 2024-11-12T12:26:54,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742355_1531 (size=31058) 2024-11-12T12:26:54,517 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/f33c3fe469774293849b335e9cef2006 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f33c3fe469774293849b335e9cef2006 2024-11-12T12:26:54,522 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/A of de303568766865fcf42f0cc60107a910 into f33c3fe469774293849b335e9cef2006(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:54,522 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:54,522 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/A, priority=13, startTime=1731414414456; duration=0sec 2024-11-12T12:26:54,522 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:54,522 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:A 2024-11-12T12:26:54,524 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:54,524 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/676a61cc0a8c4afdbfc16ef0bc7b201a is 50, key is test_row_0/C:col10/1731414410644/Put/seqid=0 2024-11-12T12:26:54,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742356_1532 (size=12104) 2024-11-12T12:26:54,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T12:26:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:54,566 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-12T12:26:54,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111258f824404e01423d9d0a433b52cd10a7_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414412816/Put/seqid=0 2024-11-12T12:26:54,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742357_1533 (size=12154) 2024-11-12T12:26:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T12:26:54,932 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/676a61cc0a8c4afdbfc16ef0bc7b201a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/676a61cc0a8c4afdbfc16ef0bc7b201a 2024-11-12T12:26:54,937 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/C of de303568766865fcf42f0cc60107a910 into 676a61cc0a8c4afdbfc16ef0bc7b201a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:54,937 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:54,937 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/C, priority=13, startTime=1731414414456; duration=0sec 2024-11-12T12:26:54,937 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:54,937 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:C 2024-11-12T12:26:54,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:26:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414474979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414474981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414474981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414474981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:54,983 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,983 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:54,983 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:54,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414474983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:54,989 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111258f824404e01423d9d0a433b52cd10a7_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111258f824404e01423d9d0a433b52cd10a7_de303568766865fcf42f0cc60107a910 2024-11-12T12:26:54,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/aad29d0ae1014d85bb998ae127b604ac, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:54,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/aad29d0ae1014d85bb998ae127b604ac is 175, key is test_row_0/A:col10/1731414412816/Put/seqid=0 2024-11-12T12:26:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742358_1534 (size=30955) 2024-11-12T12:26:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414475091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414475091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:55,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414475301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414475301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:55,393 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/aad29d0ae1014d85bb998ae127b604ac 2024-11-12T12:26:55,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/a74a0651e39d44eda1654ca3bdeac904 is 50, key is test_row_0/B:col10/1731414412816/Put/seqid=0 2024-11-12T12:26:55,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742359_1535 (size=12001) 2024-11-12T12:26:55,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:55,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414475611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414475611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T12:26:55,803 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/a74a0651e39d44eda1654ca3bdeac904 2024-11-12T12:26:55,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/7328bcb4515e490e98010dd8b99c7ae7 is 50, key is test_row_0/C:col10/1731414412816/Put/seqid=0 2024-11-12T12:26:55,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742360_1536 (size=12001) 2024-11-12T12:26:56,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414476121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:56,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414476121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:56,212 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/7328bcb4515e490e98010dd8b99c7ae7 2024-11-12T12:26:56,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/aad29d0ae1014d85bb998ae127b604ac as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/aad29d0ae1014d85bb998ae127b604ac 2024-11-12T12:26:56,218 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/aad29d0ae1014d85bb998ae127b604ac, entries=150, sequenceid=79, filesize=30.2 K 2024-11-12T12:26:56,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/a74a0651e39d44eda1654ca3bdeac904 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a74a0651e39d44eda1654ca3bdeac904 2024-11-12T12:26:56,221 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a74a0651e39d44eda1654ca3bdeac904, entries=150, sequenceid=79, filesize=11.7 K 2024-11-12T12:26:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/7328bcb4515e490e98010dd8b99c7ae7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7328bcb4515e490e98010dd8b99c7ae7 2024-11-12T12:26:56,224 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7328bcb4515e490e98010dd8b99c7ae7, entries=150, sequenceid=79, filesize=11.7 K 2024-11-12T12:26:56,225 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for de303568766865fcf42f0cc60107a910 in 1659ms, sequenceid=79, compaction requested=false 2024-11-12T12:26:56,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:56,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:56,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-12T12:26:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-12T12:26:56,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-12T12:26:56,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5820 sec 2024-11-12T12:26:56,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.5870 sec 2024-11-12T12:26:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:26:56,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:56,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-12T12:26:56,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d2134d136dd6495ca9a7f300e441bec4_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414414982/Put/seqid=0 2024-11-12T12:26:57,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742361_1537 (size=17034) 2024-11-12T12:26:57,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414477028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414477029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414477030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414477131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,134 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414477141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414477141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,143 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414477141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414477141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,286 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T12:26:57,286 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T12:26:57,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414477351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414477351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414477351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,422 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:57,424 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d2134d136dd6495ca9a7f300e441bec4_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d2134d136dd6495ca9a7f300e441bec4_de303568766865fcf42f0cc60107a910 2024-11-12T12:26:57,425 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/cfebd976906245e7b512b80add5af0a5, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:57,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/cfebd976906245e7b512b80add5af0a5 is 175, key is test_row_0/A:col10/1731414414982/Put/seqid=0 2024-11-12T12:26:57,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742362_1538 (size=48139) 2024-11-12T12:26:57,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/cfebd976906245e7b512b80add5af0a5 2024-11-12T12:26:57,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/1db140d1a75245969d64e2600b8a0346 is 50, key is test_row_0/B:col10/1731414414982/Put/seqid=0 2024-11-12T12:26:57,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742363_1539 (size=12001) 2024-11-12T12:26:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414477661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414477661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414477661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T12:26:57,811 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:57,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-12T12:26:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T12:26:57,813 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:57,814 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:57,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:26:57,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/1db140d1a75245969d64e2600b8a0346 2024-11-12T12:26:57,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/7c6512c1a3264ccd8cead5976e11cd1d is 50, key is test_row_0/C:col10/1731414414982/Put/seqid=0 2024-11-12T12:26:57,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742364_1540 (size=12001) 2024-11-12T12:26:57,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T12:26:57,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T12:26:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:57,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:57,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:58,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T12:26:58,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:58,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:58,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:58,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:58,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:26:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T12:26:58,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414478171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:58,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414478171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:58,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414478171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:58,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/7c6512c1a3264ccd8cead5976e11cd1d 2024-11-12T12:26:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/cfebd976906245e7b512b80add5af0a5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/cfebd976906245e7b512b80add5af0a5 2024-11-12T12:26:58,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/cfebd976906245e7b512b80add5af0a5, entries=250, sequenceid=93, filesize=47.0 K 2024-11-12T12:26:58,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/1db140d1a75245969d64e2600b8a0346 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1db140d1a75245969d64e2600b8a0346 2024-11-12T12:26:58,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1db140d1a75245969d64e2600b8a0346, entries=150, sequenceid=93, filesize=11.7 K 2024-11-12T12:26:58,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/7c6512c1a3264ccd8cead5976e11cd1d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7c6512c1a3264ccd8cead5976e11cd1d 2024-11-12T12:26:58,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7c6512c1a3264ccd8cead5976e11cd1d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-12T12:26:58,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for de303568766865fcf42f0cc60107a910 in 1266ms, sequenceid=93, compaction requested=true 2024-11-12T12:26:58,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:58,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:26:58,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:58,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:26:58,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:58,260 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:58,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:26:58,260 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:58,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:58,260 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:58,260 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:58,260 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/B is initiating minor compaction (all files) 2024-11-12T12:26:58,260 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/A is initiating minor compaction (all files) 2024-11-12T12:26:58,260 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/A in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:58,260 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/B in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:58,260 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6984b1eb67d849ed8be0582d895da3b7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a74a0651e39d44eda1654ca3bdeac904, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1db140d1a75245969d64e2600b8a0346] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=35.3 K 2024-11-12T12:26:58,260 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f33c3fe469774293849b335e9cef2006, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/aad29d0ae1014d85bb998ae127b604ac, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/cfebd976906245e7b512b80add5af0a5] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=107.6 K 2024-11-12T12:26:58,260 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:58,260 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f33c3fe469774293849b335e9cef2006, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/aad29d0ae1014d85bb998ae127b604ac, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/cfebd976906245e7b512b80add5af0a5] 2024-11-12T12:26:58,261 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6984b1eb67d849ed8be0582d895da3b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414410644 2024-11-12T12:26:58,261 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting f33c3fe469774293849b335e9cef2006, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414410644 2024-11-12T12:26:58,261 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a74a0651e39d44eda1654ca3bdeac904, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731414412816 2024-11-12T12:26:58,261 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad29d0ae1014d85bb998ae127b604ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731414412816 2024-11-12T12:26:58,261 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 1db140d1a75245969d64e2600b8a0346, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731414414967 2024-11-12T12:26:58,261 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfebd976906245e7b512b80add5af0a5, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731414414967 2024-11-12T12:26:58,265 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:58,266 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:58,266 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/697a18c4886d40efbf06d3c901debb4c is 50, key is test_row_0/B:col10/1731414414982/Put/seqid=0 2024-11-12T12:26:58,266 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112859e3082569745e2bc0c4aa7f5a1729d_de303568766865fcf42f0cc60107a910 store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:58,268 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112859e3082569745e2bc0c4aa7f5a1729d_de303568766865fcf42f0cc60107a910, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:58,268 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112859e3082569745e2bc0c4aa7f5a1729d_de303568766865fcf42f0cc60107a910 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:58,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T12:26:58,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:58,271 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-12T12:26:58,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742365_1541 (size=12207) 2024-11-12T12:26:58,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742366_1542 (size=4469) 2024-11-12T12:26:58,275 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#A#compaction#457 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:58,275 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/0047aa77dc394e30b1d6f52d9dfdd7b5 is 175, key is test_row_0/A:col10/1731414414982/Put/seqid=0 2024-11-12T12:26:58,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120e06b986462449278b96b6fd5a10f37c_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414417028/Put/seqid=0 2024-11-12T12:26:58,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742367_1543 (size=31161) 2024-11-12T12:26:58,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742368_1544 (size=12154) 2024-11-12T12:26:58,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:26:58,284 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120e06b986462449278b96b6fd5a10f37c_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120e06b986462449278b96b6fd5a10f37c_de303568766865fcf42f0cc60107a910 2024-11-12T12:26:58,285 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/0047aa77dc394e30b1d6f52d9dfdd7b5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0047aa77dc394e30b1d6f52d9dfdd7b5 2024-11-12T12:26:58,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/8bc54731096d4567a5e34216f6889db6, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:26:58,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/8bc54731096d4567a5e34216f6889db6 is 175, key is test_row_0/A:col10/1731414417028/Put/seqid=0 2024-11-12T12:26:58,288 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/A of de303568766865fcf42f0cc60107a910 into 0047aa77dc394e30b1d6f52d9dfdd7b5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:58,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742369_1545 (size=30955) 2024-11-12T12:26:58,288 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:58,289 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/A, priority=13, startTime=1731414418259; duration=0sec 2024-11-12T12:26:58,289 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:26:58,289 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:A 2024-11-12T12:26:58,289 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:26:58,289 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:26:58,289 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/C is initiating minor compaction (all files) 2024-11-12T12:26:58,289 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/C in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:58,289 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/676a61cc0a8c4afdbfc16ef0bc7b201a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7328bcb4515e490e98010dd8b99c7ae7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7c6512c1a3264ccd8cead5976e11cd1d] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=35.3 K 2024-11-12T12:26:58,290 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 676a61cc0a8c4afdbfc16ef0bc7b201a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731414410644 2024-11-12T12:26:58,290 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7328bcb4515e490e98010dd8b99c7ae7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731414412816 2024-11-12T12:26:58,290 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6512c1a3264ccd8cead5976e11cd1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731414414967 2024-11-12T12:26:58,295 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:26:58,296 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/d6af290ae7e5408a9888c37e91ae2de8 is 50, key is test_row_0/C:col10/1731414414982/Put/seqid=0 2024-11-12T12:26:58,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742370_1546 (size=12207) 2024-11-12T12:26:58,302 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/d6af290ae7e5408a9888c37e91ae2de8 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/d6af290ae7e5408a9888c37e91ae2de8 2024-11-12T12:26:58,305 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/C of de303568766865fcf42f0cc60107a910 into d6af290ae7e5408a9888c37e91ae2de8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:58,305 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:58,305 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/C, priority=13, startTime=1731414418260; duration=0sec 2024-11-12T12:26:58,305 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:58,305 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:C 2024-11-12T12:26:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T12:26:58,677 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/697a18c4886d40efbf06d3c901debb4c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/697a18c4886d40efbf06d3c901debb4c 2024-11-12T12:26:58,680 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/B of de303568766865fcf42f0cc60107a910 into 697a18c4886d40efbf06d3c901debb4c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:26:58,680 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:58,680 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/B, priority=13, startTime=1731414418260; duration=0sec 2024-11-12T12:26:58,680 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:26:58,680 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:B 2024-11-12T12:26:58,689 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/8bc54731096d4567a5e34216f6889db6 2024-11-12T12:26:58,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/6b92b3b0d5b843e49dd49194cf25dd6d is 50, key is test_row_0/B:col10/1731414417028/Put/seqid=0 2024-11-12T12:26:58,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742371_1547 (size=12001) 2024-11-12T12:26:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T12:26:59,100 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/6b92b3b0d5b843e49dd49194cf25dd6d 2024-11-12T12:26:59,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/c788818d611e41409a25f72de6fe8e15 is 50, key is test_row_0/C:col10/1731414417028/Put/seqid=0 2024-11-12T12:26:59,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742372_1548 (size=12001) 2024-11-12T12:26:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:26:59,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:26:59,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414479166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414479168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414479181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414479181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414479181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,183 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,184 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,184 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:26:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414479270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414479281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414479481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414479491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,508 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/c788818d611e41409a25f72de6fe8e15 2024-11-12T12:26:59,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/8bc54731096d4567a5e34216f6889db6 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/8bc54731096d4567a5e34216f6889db6 2024-11-12T12:26:59,514 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/8bc54731096d4567a5e34216f6889db6, entries=150, sequenceid=116, filesize=30.2 K 2024-11-12T12:26:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/6b92b3b0d5b843e49dd49194cf25dd6d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6b92b3b0d5b843e49dd49194cf25dd6d 2024-11-12T12:26:59,518 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6b92b3b0d5b843e49dd49194cf25dd6d, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T12:26:59,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/c788818d611e41409a25f72de6fe8e15 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/c788818d611e41409a25f72de6fe8e15 2024-11-12T12:26:59,521 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/c788818d611e41409a25f72de6fe8e15, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T12:26:59,521 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for de303568766865fcf42f0cc60107a910 in 1250ms, sequenceid=116, compaction requested=false 2024-11-12T12:26:59,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:26:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:26:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-12T12:26:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-12T12:26:59,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-12T12:26:59,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7090 sec 2024-11-12T12:26:59,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7130 sec 2024-11-12T12:26:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:26:59,793 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:26:59,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-12T12:26:59,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123e91c55352a34198b6abea67876e9962_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414419164/Put/seqid=0 2024-11-12T12:26:59,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742373_1549 (size=14744) 2024-11-12T12:26:59,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414479832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414479834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414479941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:26:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414479941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:26:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:26:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:26:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T12:26:59,962 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:26:59,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:26:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-12T12:26:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T12:26:59,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:26:59,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:26:59,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:27:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T12:27:00,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414480151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:00,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414480151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:00,202 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:27:00,205 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123e91c55352a34198b6abea67876e9962_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123e91c55352a34198b6abea67876e9962_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:00,205 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/22b22cdf853a4ed398ce23148ee17ee0, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:00,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/22b22cdf853a4ed398ce23148ee17ee0 is 175, key is test_row_0/A:col10/1731414419164/Put/seqid=0 2024-11-12T12:27:00,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742374_1550 (size=39699) 2024-11-12T12:27:00,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:00,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:00,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T12:27:00,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:00,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:00,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414480460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:00,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414480461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:00,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T12:27:00,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/22b22cdf853a4ed398ce23148ee17ee0 2024-11-12T12:27:00,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/1c49575f8fe745d4b26f259a25c49f3c is 50, key is test_row_0/B:col10/1731414419164/Put/seqid=0 2024-11-12T12:27:00,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742375_1551 (size=12151) 2024-11-12T12:27:00,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:00,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:00,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:00,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:00,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:00,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414480971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414480971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:00,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:01,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/1c49575f8fe745d4b26f259a25c49f3c 2024-11-12T12:27:01,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/b7c9d4aae2aa4caa8d4427b47a907d8f is 50, key is test_row_0/C:col10/1731414419164/Put/seqid=0 2024-11-12T12:27:01,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742376_1552 (size=12151) 2024-11-12T12:27:01,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:01,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T12:27:01,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:01,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414481201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414481201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414481201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:01,204 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:27:01,204 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:27:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:01,204 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_2' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:27:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:01,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:01,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:01,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:01,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:01,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/b7c9d4aae2aa4caa8d4427b47a907d8f 2024-11-12T12:27:01,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/22b22cdf853a4ed398ce23148ee17ee0 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/22b22cdf853a4ed398ce23148ee17ee0 2024-11-12T12:27:01,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/22b22cdf853a4ed398ce23148ee17ee0, entries=200, sequenceid=133, filesize=38.8 K 2024-11-12T12:27:01,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/1c49575f8fe745d4b26f259a25c49f3c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1c49575f8fe745d4b26f259a25c49f3c 2024-11-12T12:27:01,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1c49575f8fe745d4b26f259a25c49f3c, entries=150, sequenceid=133, filesize=11.9 K 2024-11-12T12:27:01,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/b7c9d4aae2aa4caa8d4427b47a907d8f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b7c9d4aae2aa4caa8d4427b47a907d8f 2024-11-12T12:27:01,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b7c9d4aae2aa4caa8d4427b47a907d8f, entries=150, sequenceid=133, filesize=11.9 K 2024-11-12T12:27:01,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for de303568766865fcf42f0cc60107a910 in 1647ms, sequenceid=133, compaction requested=true 2024-11-12T12:27:01,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:27:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:01,441 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:27:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:01,441 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:27:01,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:27:01,441 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:01,441 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:01,441 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/B is initiating minor compaction (all files) 2024-11-12T12:27:01,441 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/A is initiating minor compaction (all files) 2024-11-12T12:27:01,441 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/B in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,441 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/A in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,441 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/697a18c4886d40efbf06d3c901debb4c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6b92b3b0d5b843e49dd49194cf25dd6d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1c49575f8fe745d4b26f259a25c49f3c] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=35.5 K 2024-11-12T12:27:01,441 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0047aa77dc394e30b1d6f52d9dfdd7b5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/8bc54731096d4567a5e34216f6889db6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/22b22cdf853a4ed398ce23148ee17ee0] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=99.4 K 2024-11-12T12:27:01,442 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,442 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0047aa77dc394e30b1d6f52d9dfdd7b5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/8bc54731096d4567a5e34216f6889db6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/22b22cdf853a4ed398ce23148ee17ee0] 2024-11-12T12:27:01,442 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 697a18c4886d40efbf06d3c901debb4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731414414967 2024-11-12T12:27:01,442 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0047aa77dc394e30b1d6f52d9dfdd7b5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731414414967 2024-11-12T12:27:01,442 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b92b3b0d5b843e49dd49194cf25dd6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731414417024 2024-11-12T12:27:01,442 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc54731096d4567a5e34216f6889db6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731414417024 2024-11-12T12:27:01,442 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c49575f8fe745d4b26f259a25c49f3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414419164 2024-11-12T12:27:01,442 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b22cdf853a4ed398ce23148ee17ee0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414419155 2024-11-12T12:27:01,446 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:01,447 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:01,448 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/39a7bed0bbc849c1842dfc37b0cd8ca2 is 50, key is test_row_0/B:col10/1731414419164/Put/seqid=0 2024-11-12T12:27:01,453 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112f52572ee2c0d41568425e3af000ac705_de303568766865fcf42f0cc60107a910 store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:01,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742377_1553 (size=12459) 2024-11-12T12:27:01,455 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112f52572ee2c0d41568425e3af000ac705_de303568766865fcf42f0cc60107a910, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:01,455 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f52572ee2c0d41568425e3af000ac705_de303568766865fcf42f0cc60107a910 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:01,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742378_1554 (size=4469) 2024-11-12T12:27:01,459 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/39a7bed0bbc849c1842dfc37b0cd8ca2 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/39a7bed0bbc849c1842dfc37b0cd8ca2 2024-11-12T12:27:01,462 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/B of de303568766865fcf42f0cc60107a910 into 39a7bed0bbc849c1842dfc37b0cd8ca2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:01,462 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:01,462 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/B, priority=13, startTime=1731414421441; duration=0sec 2024-11-12T12:27:01,462 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:27:01,462 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:B 2024-11-12T12:27:01,462 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:01,463 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:01,463 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/C is initiating minor compaction (all files) 2024-11-12T12:27:01,463 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/C in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,463 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/d6af290ae7e5408a9888c37e91ae2de8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/c788818d611e41409a25f72de6fe8e15, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b7c9d4aae2aa4caa8d4427b47a907d8f] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=35.5 K 2024-11-12T12:27:01,463 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d6af290ae7e5408a9888c37e91ae2de8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731414414967 2024-11-12T12:27:01,464 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting c788818d611e41409a25f72de6fe8e15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731414417024 2024-11-12T12:27:01,464 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting b7c9d4aae2aa4caa8d4427b47a907d8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414419164 2024-11-12T12:27:01,468 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:01,468 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/6e62183bba944282b04f8af4990aaf52 is 50, key is test_row_0/C:col10/1731414419164/Put/seqid=0 2024-11-12T12:27:01,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742379_1555 (size=12459) 2024-11-12T12:27:01,474 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/6e62183bba944282b04f8af4990aaf52 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e62183bba944282b04f8af4990aaf52 2024-11-12T12:27:01,477 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/C of de303568766865fcf42f0cc60107a910 into 6e62183bba944282b04f8af4990aaf52(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:01,477 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:01,477 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/C, priority=13, startTime=1731414421441; duration=0sec 2024-11-12T12:27:01,477 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:01,477 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:C 2024-11-12T12:27:01,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T12:27:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:01,488 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-12T12:27:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112634e163bf0014ca194f8f4bcaae81107_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414419833/Put/seqid=0 2024-11-12T12:27:01,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742380_1556 (size=12304) 2024-11-12T12:27:01,859 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:01,860 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/568a4e80123d4baba6f430bccb0e733f is 175, key is test_row_0/A:col10/1731414419164/Put/seqid=0 2024-11-12T12:27:01,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742381_1557 (size=31413) 2024-11-12T12:27:01,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:27:01,899 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112634e163bf0014ca194f8f4bcaae81107_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112634e163bf0014ca194f8f4bcaae81107_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/bd463f724ff04f35bf493a6129b94470, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/bd463f724ff04f35bf493a6129b94470 is 175, key is test_row_0/A:col10/1731414419833/Put/seqid=0 2024-11-12T12:27:01,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742382_1558 (size=31105) 2024-11-12T12:27:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:27:01,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:02,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414482010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:02,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414482014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T12:27:02,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414482121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:02,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414482121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:02,266 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/568a4e80123d4baba6f430bccb0e733f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/568a4e80123d4baba6f430bccb0e733f 2024-11-12T12:27:02,269 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/A of de303568766865fcf42f0cc60107a910 into 568a4e80123d4baba6f430bccb0e733f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:02,269 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:02,269 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/A, priority=13, startTime=1731414421440; duration=0sec 2024-11-12T12:27:02,270 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:02,270 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:A 2024-11-12T12:27:02,303 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/bd463f724ff04f35bf493a6129b94470 2024-11-12T12:27:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/d3ef135c8a5941d8b52d018efa9b97cc is 50, key is test_row_0/B:col10/1731414419833/Put/seqid=0 2024-11-12T12:27:02,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742383_1559 (size=12151) 2024-11-12T12:27:02,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414482331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:02,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414482331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414482641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414482641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:02,711 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/d3ef135c8a5941d8b52d018efa9b97cc 2024-11-12T12:27:02,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/772bbd4724694f7baa08690fe7d2d2ff is 50, key is test_row_0/C:col10/1731414419833/Put/seqid=0 2024-11-12T12:27:02,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742384_1560 (size=12151) 2024-11-12T12:27:02,720 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/772bbd4724694f7baa08690fe7d2d2ff 2024-11-12T12:27:02,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/bd463f724ff04f35bf493a6129b94470 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/bd463f724ff04f35bf493a6129b94470 2024-11-12T12:27:02,726 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/bd463f724ff04f35bf493a6129b94470, entries=150, sequenceid=157, filesize=30.4 K 2024-11-12T12:27:02,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/d3ef135c8a5941d8b52d018efa9b97cc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d3ef135c8a5941d8b52d018efa9b97cc 2024-11-12T12:27:02,734 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d3ef135c8a5941d8b52d018efa9b97cc, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T12:27:02,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/772bbd4724694f7baa08690fe7d2d2ff as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/772bbd4724694f7baa08690fe7d2d2ff 2024-11-12T12:27:02,737 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/772bbd4724694f7baa08690fe7d2d2ff, entries=150, sequenceid=157, filesize=11.9 K 2024-11-12T12:27:02,738 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for de303568766865fcf42f0cc60107a910 in 1250ms, sequenceid=157, compaction requested=false 2024-11-12T12:27:02,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:02,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:02,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-12T12:27:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-12T12:27:02,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-12T12:27:02,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7750 sec 2024-11-12T12:27:02,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.7790 sec 2024-11-12T12:27:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:27:03,153 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:27:03,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-12T12:27:03,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126e866b3586a4424bbd0638d400c0fe9f_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414423153/Put/seqid=0 2024-11-12T12:27:03,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742385_1561 (size=14794) 2024-11-12T12:27:03,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414483190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:03,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:03,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414483194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:03,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414483301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414483301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:03,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414483511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:03,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414483511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:03,563 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:27:03,566 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126e866b3586a4424bbd0638d400c0fe9f_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126e866b3586a4424bbd0638d400c0fe9f_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:03,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d07627e641cd4c278d391cec18d4d50c, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:03,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d07627e641cd4c278d391cec18d4d50c is 175, key is test_row_0/A:col10/1731414423153/Put/seqid=0 2024-11-12T12:27:03,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742386_1562 (size=39749) 2024-11-12T12:27:03,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414483821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:03,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414483821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:03,970 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d07627e641cd4c278d391cec18d4d50c 2024-11-12T12:27:03,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/bdc882e71897432292537fadf0758679 is 50, key is test_row_0/B:col10/1731414423153/Put/seqid=0 2024-11-12T12:27:03,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742387_1563 (size=12151) 2024-11-12T12:27:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T12:27:04,121 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:27:04,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:27:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-12T12:27:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T12:27:04,123 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:27:04,123 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:27:04,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:27:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T12:27:04,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T12:27:04,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:04,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:04,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:04,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:04,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:04,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414484331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:04,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414484331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:04,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/bdc882e71897432292537fadf0758679 2024-11-12T12:27:04,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/6e7e7588e99c43609081d80fb8b7beb3 is 50, key is test_row_0/C:col10/1731414423153/Put/seqid=0 2024-11-12T12:27:04,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742388_1564 (size=12151) 2024-11-12T12:27:04,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/6e7e7588e99c43609081d80fb8b7beb3 2024-11-12T12:27:04,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d07627e641cd4c278d391cec18d4d50c as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d07627e641cd4c278d391cec18d4d50c 2024-11-12T12:27:04,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d07627e641cd4c278d391cec18d4d50c, entries=200, sequenceid=174, filesize=38.8 K 2024-11-12T12:27:04,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/bdc882e71897432292537fadf0758679 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/bdc882e71897432292537fadf0758679 2024-11-12T12:27:04,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/bdc882e71897432292537fadf0758679, entries=150, sequenceid=174, filesize=11.9 K 2024-11-12T12:27:04,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/6e7e7588e99c43609081d80fb8b7beb3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e7e7588e99c43609081d80fb8b7beb3 2024-11-12T12:27:04,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e7e7588e99c43609081d80fb8b7beb3, entries=150, sequenceid=174, filesize=11.9 K 2024-11-12T12:27:04,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for de303568766865fcf42f0cc60107a910 in 1255ms, sequenceid=174, compaction requested=true 2024-11-12T12:27:04,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:04,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:27:04,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:04,409 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:04,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:27:04,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:27:04,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:27:04,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T12:27:04,412 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:04,412 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:04,413 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/A is initiating minor compaction (all files) 2024-11-12T12:27:04,413 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/A in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:04,413 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/568a4e80123d4baba6f430bccb0e733f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/bd463f724ff04f35bf493a6129b94470, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d07627e641cd4c278d391cec18d4d50c] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=99.9 K 2024-11-12T12:27:04,413 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:04,413 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/568a4e80123d4baba6f430bccb0e733f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/bd463f724ff04f35bf493a6129b94470, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d07627e641cd4c278d391cec18d4d50c] 2024-11-12T12:27:04,413 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 568a4e80123d4baba6f430bccb0e733f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414419164 2024-11-12T12:27:04,413 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd463f724ff04f35bf493a6129b94470, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731414419829 2024-11-12T12:27:04,414 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting d07627e641cd4c278d391cec18d4d50c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731414422004 2024-11-12T12:27:04,414 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:04,414 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/B is initiating minor compaction (all files) 2024-11-12T12:27:04,414 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/B in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:04,414 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/39a7bed0bbc849c1842dfc37b0cd8ca2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d3ef135c8a5941d8b52d018efa9b97cc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/bdc882e71897432292537fadf0758679] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=35.9 K 2024-11-12T12:27:04,414 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 39a7bed0bbc849c1842dfc37b0cd8ca2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414419164 2024-11-12T12:27:04,415 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting d3ef135c8a5941d8b52d018efa9b97cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731414419829 2024-11-12T12:27:04,415 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting bdc882e71897432292537fadf0758679, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731414422004 2024-11-12T12:27:04,419 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:04,422 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:04,422 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/33d1b10d46564f09893ed5c1d74747f2 is 50, key is test_row_0/B:col10/1731414423153/Put/seqid=0 2024-11-12T12:27:04,423 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112e43b45dd05b04f6dac6f25d75aa4adec_de303568766865fcf42f0cc60107a910 store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:04,425 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112e43b45dd05b04f6dac6f25d75aa4adec_de303568766865fcf42f0cc60107a910, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:04,425 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e43b45dd05b04f6dac6f25d75aa4adec_de303568766865fcf42f0cc60107a910 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:04,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742389_1565 (size=12561) 2024-11-12T12:27:04,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T12:27:04,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:04,429 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-12T12:27:04,433 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/33d1b10d46564f09893ed5c1d74747f2 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/33d1b10d46564f09893ed5c1d74747f2 2024-11-12T12:27:04,438 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/B of de303568766865fcf42f0cc60107a910 into 33d1b10d46564f09893ed5c1d74747f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:04,438 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:04,438 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/B, priority=13, startTime=1731414424409; duration=0sec 2024-11-12T12:27:04,438 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:27:04,438 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:B 2024-11-12T12:27:04,438 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:04,439 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:04,439 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/C is initiating minor compaction (all files) 2024-11-12T12:27:04,439 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/C in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:04,439 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e62183bba944282b04f8af4990aaf52, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/772bbd4724694f7baa08690fe7d2d2ff, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e7e7588e99c43609081d80fb8b7beb3] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=35.9 K 2024-11-12T12:27:04,440 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e62183bba944282b04f8af4990aaf52, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731414419164 2024-11-12T12:27:04,440 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 772bbd4724694f7baa08690fe7d2d2ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731414419829 2024-11-12T12:27:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T12:27:04,441 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e7e7588e99c43609081d80fb8b7beb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731414422004 2024-11-12T12:27:04,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742390_1566 (size=4469) 2024-11-12T12:27:04,446 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#A#compaction#474 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:04,447 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/5d01163452ca41f083acb934d426598f is 175, key is test_row_0/A:col10/1731414423153/Put/seqid=0 2024-11-12T12:27:04,450 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#C#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:04,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bc010c524119467d998f4479de9f3f0d_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414423192/Put/seqid=0 2024-11-12T12:27:04,450 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/4bf502fbb04e4763a851b466af0acf57 is 50, key is test_row_0/C:col10/1731414423153/Put/seqid=0 2024-11-12T12:27:04,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742391_1567 (size=31515) 2024-11-12T12:27:04,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742392_1568 (size=12304) 2024-11-12T12:27:04,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742393_1569 (size=12561) 2024-11-12T12:27:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T12:27:04,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:27:04,864 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bc010c524119467d998f4479de9f3f0d_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bc010c524119467d998f4479de9f3f0d_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:04,874 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/5d01163452ca41f083acb934d426598f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/5d01163452ca41f083acb934d426598f 2024-11-12T12:27:04,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d22d4eb1769944989e37257549d752ef, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:04,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d22d4eb1769944989e37257549d752ef is 175, key is test_row_0/A:col10/1731414423192/Put/seqid=0 2024-11-12T12:27:04,878 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/4bf502fbb04e4763a851b466af0acf57 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/4bf502fbb04e4763a851b466af0acf57 2024-11-12T12:27:04,879 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/A of de303568766865fcf42f0cc60107a910 into 5d01163452ca41f083acb934d426598f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:04,879 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:04,879 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/A, priority=13, startTime=1731414424409; duration=0sec 2024-11-12T12:27:04,879 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:04,879 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:A 2024-11-12T12:27:04,884 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/C of de303568766865fcf42f0cc60107a910 into 4bf502fbb04e4763a851b466af0acf57(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:04,884 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:04,884 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/C, priority=13, startTime=1731414424409; duration=0sec 2024-11-12T12:27:04,884 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:04,884 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:C 2024-11-12T12:27:04,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742394_1570 (size=31105) 2024-11-12T12:27:04,917 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d22d4eb1769944989e37257549d752ef 2024-11-12T12:27:04,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/2bf0a3966a7a43f387388d1b4a0a9c0a is 50, key is test_row_0/B:col10/1731414423192/Put/seqid=0 2024-11-12T12:27:04,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742395_1571 (size=12151) 2024-11-12T12:27:05,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:27:05,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414485239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414485242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414485242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T12:27:05,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414485341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414485341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,343 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:27:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,343 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:27:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,347 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/2bf0a3966a7a43f387388d1b4a0a9c0a 2024-11-12T12:27:05,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/70d7801d86654c9b850f54e2334598c7 is 50, key is test_row_0/C:col10/1731414423192/Put/seqid=0 2024-11-12T12:27:05,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414485351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414485351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414485361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742396_1572 (size=12151) 2024-11-12T12:27:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414485561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414485561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414485571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,770 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/70d7801d86654c9b850f54e2334598c7 2024-11-12T12:27:05,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d22d4eb1769944989e37257549d752ef as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d22d4eb1769944989e37257549d752ef 2024-11-12T12:27:05,777 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d22d4eb1769944989e37257549d752ef, entries=150, sequenceid=194, filesize=30.4 K 2024-11-12T12:27:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/2bf0a3966a7a43f387388d1b4a0a9c0a as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/2bf0a3966a7a43f387388d1b4a0a9c0a 2024-11-12T12:27:05,789 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/2bf0a3966a7a43f387388d1b4a0a9c0a, entries=150, sequenceid=194, filesize=11.9 K 2024-11-12T12:27:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/70d7801d86654c9b850f54e2334598c7 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/70d7801d86654c9b850f54e2334598c7 2024-11-12T12:27:05,793 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/70d7801d86654c9b850f54e2334598c7, entries=150, sequenceid=194, filesize=11.9 K 2024-11-12T12:27:05,794 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for de303568766865fcf42f0cc60107a910 in 1365ms, sequenceid=194, compaction requested=false 2024-11-12T12:27:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-12T12:27:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-12T12:27:05,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-12T12:27:05,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6730 sec 2024-11-12T12:27:05,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.6880 sec 2024-11-12T12:27:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:27:05,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:27:05,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-12T12:27:05,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111235a73eed52814e0d8d2f1416fcb23af1_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414425873/Put/seqid=0 2024-11-12T12:27:05,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742397_1573 (size=14794) 2024-11-12T12:27:05,902 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:27:05,904 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111235a73eed52814e0d8d2f1416fcb23af1_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111235a73eed52814e0d8d2f1416fcb23af1_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:05,905 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d343a53b791942f5b24b01af9b75f855, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:05,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d343a53b791942f5b24b01af9b75f855 is 175, key is test_row_0/A:col10/1731414425873/Put/seqid=0 2024-11-12T12:27:05,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742398_1574 (size=39749) 2024-11-12T12:27:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414485942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414485942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414485942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414486051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414486051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:06,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414486051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:06,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414486261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:06,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414486261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414486261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T12:27:06,271 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:27:06,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:27:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-12T12:27:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T12:27:06,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:27:06,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:27:06,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:27:06,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d343a53b791942f5b24b01af9b75f855 2024-11-12T12:27:06,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/25cf2df45b4548ac8c3e44ec01512a81 is 50, key is test_row_0/B:col10/1731414425873/Put/seqid=0 2024-11-12T12:27:06,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742399_1575 (size=12151) 2024-11-12T12:27:06,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/25cf2df45b4548ac8c3e44ec01512a81 2024-11-12T12:27:06,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/9bb38b016fae4791aa3be72d820d3e27 is 50, key is test_row_0/C:col10/1731414425873/Put/seqid=0 2024-11-12T12:27:06,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742400_1576 (size=12151) 2024-11-12T12:27:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T12:27:06,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T12:27:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:06,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:06,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414486571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:06,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414486571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:06,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414486571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:06,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T12:27:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T12:27:06,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T12:27:06,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:06,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:06,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/9bb38b016fae4791aa3be72d820d3e27 2024-11-12T12:27:06,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/d343a53b791942f5b24b01af9b75f855 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d343a53b791942f5b24b01af9b75f855 2024-11-12T12:27:06,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d343a53b791942f5b24b01af9b75f855, entries=200, sequenceid=215, filesize=38.8 K 2024-11-12T12:27:06,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/25cf2df45b4548ac8c3e44ec01512a81 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/25cf2df45b4548ac8c3e44ec01512a81 2024-11-12T12:27:06,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/25cf2df45b4548ac8c3e44ec01512a81, entries=150, sequenceid=215, filesize=11.9 K 2024-11-12T12:27:06,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/9bb38b016fae4791aa3be72d820d3e27 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/9bb38b016fae4791aa3be72d820d3e27 2024-11-12T12:27:06,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/9bb38b016fae4791aa3be72d820d3e27, entries=150, sequenceid=215, filesize=11.9 K 2024-11-12T12:27:06,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for de303568766865fcf42f0cc60107a910 in 895ms, sequenceid=215, compaction requested=true 2024-11-12T12:27:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:27:06,769 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:06,769 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:27:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:27:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:27:06,770 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:06,770 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:06,770 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/B is initiating minor compaction (all files) 2024-11-12T12:27:06,770 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/A is initiating minor compaction (all files) 2024-11-12T12:27:06,770 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/A in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,770 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/B in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,770 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/33d1b10d46564f09893ed5c1d74747f2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/2bf0a3966a7a43f387388d1b4a0a9c0a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/25cf2df45b4548ac8c3e44ec01512a81] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=36.0 K 2024-11-12T12:27:06,770 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/5d01163452ca41f083acb934d426598f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d22d4eb1769944989e37257549d752ef, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d343a53b791942f5b24b01af9b75f855] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=100.0 K 2024-11-12T12:27:06,770 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,770 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/5d01163452ca41f083acb934d426598f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d22d4eb1769944989e37257549d752ef, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d343a53b791942f5b24b01af9b75f855] 2024-11-12T12:27:06,772 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d1b10d46564f09893ed5c1d74747f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731414422004 2024-11-12T12:27:06,772 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d01163452ca41f083acb934d426598f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731414422004 2024-11-12T12:27:06,772 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf0a3966a7a43f387388d1b4a0a9c0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731414423183 2024-11-12T12:27:06,772 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting d22d4eb1769944989e37257549d752ef, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731414423183 2024-11-12T12:27:06,772 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 25cf2df45b4548ac8c3e44ec01512a81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414425237 2024-11-12T12:27:06,772 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting d343a53b791942f5b24b01af9b75f855, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414425237 2024-11-12T12:27:06,777 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:06,781 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112a9588679ae6b482caa684fcf994d04db_de303568766865fcf42f0cc60107a910 store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:06,782 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:06,782 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/43b841e3d5ec4296a7c22aa4a786cebb is 50, key is test_row_0/B:col10/1731414425873/Put/seqid=0 2024-11-12T12:27:06,783 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112a9588679ae6b482caa684fcf994d04db_de303568766865fcf42f0cc60107a910, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:06,783 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a9588679ae6b482caa684fcf994d04db_de303568766865fcf42f0cc60107a910 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:06,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742401_1577 (size=12663) 2024-11-12T12:27:06,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742402_1578 (size=4469) 2024-11-12T12:27:06,806 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#A#compaction#483 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:06,807 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/0f6748a34b784d30a24521e2141354ee is 175, key is test_row_0/A:col10/1731414425873/Put/seqid=0 2024-11-12T12:27:06,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742403_1579 (size=31617) 2024-11-12T12:27:06,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T12:27:06,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:06,886 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-12T12:27:06,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122c4dc999b8df442aa01c24acbfac6bc8_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414425910/Put/seqid=0 2024-11-12T12:27:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T12:27:06,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742404_1580 (size=12304) 2024-11-12T12:27:06,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:27:06,917 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122c4dc999b8df442aa01c24acbfac6bc8_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122c4dc999b8df442aa01c24acbfac6bc8_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:06,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/eea47d0745df48a99fda6bf97769d060, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:06,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/eea47d0745df48a99fda6bf97769d060 is 175, key is test_row_0/A:col10/1731414425910/Put/seqid=0 2024-11-12T12:27:06,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742405_1581 (size=31105) 2024-11-12T12:27:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:27:07,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:07,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414487100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414487105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414487105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,201 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/43b841e3d5ec4296a7c22aa4a786cebb as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/43b841e3d5ec4296a7c22aa4a786cebb 2024-11-12T12:27:07,205 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/B of de303568766865fcf42f0cc60107a910 into 43b841e3d5ec4296a7c22aa4a786cebb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:07,205 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:07,205 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/B, priority=13, startTime=1731414426769; duration=0sec 2024-11-12T12:27:07,205 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:27:07,205 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:B 2024-11-12T12:27:07,205 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:07,206 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:07,206 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/C is initiating minor compaction (all files) 2024-11-12T12:27:07,206 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/C in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:07,206 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/4bf502fbb04e4763a851b466af0acf57, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/70d7801d86654c9b850f54e2334598c7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/9bb38b016fae4791aa3be72d820d3e27] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=36.0 K 2024-11-12T12:27:07,206 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf502fbb04e4763a851b466af0acf57, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731414422004 2024-11-12T12:27:07,206 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 70d7801d86654c9b850f54e2334598c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731414423183 2024-11-12T12:27:07,207 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb38b016fae4791aa3be72d820d3e27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414425237 2024-11-12T12:27:07,213 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:07,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414487211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,214 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/31bbf446f0ab4952bb0ae581435e350d is 50, key is test_row_0/C:col10/1731414425873/Put/seqid=0 2024-11-12T12:27:07,218 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/0f6748a34b784d30a24521e2141354ee as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0f6748a34b784d30a24521e2141354ee 2024-11-12T12:27:07,222 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/A of de303568766865fcf42f0cc60107a910 into 0f6748a34b784d30a24521e2141354ee(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:07,222 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:07,222 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/A, priority=13, startTime=1731414426769; duration=0sec 2024-11-12T12:27:07,222 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:07,222 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:A 2024-11-12T12:27:07,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414487221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414487221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742406_1582 (size=12663) 2024-11-12T12:27:07,231 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/31bbf446f0ab4952bb0ae581435e350d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/31bbf446f0ab4952bb0ae581435e350d 2024-11-12T12:27:07,234 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/C of de303568766865fcf42f0cc60107a910 into 31bbf446f0ab4952bb0ae581435e350d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:07,234 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:07,234 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/C, priority=13, startTime=1731414426769; duration=0sec 2024-11-12T12:27:07,234 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:07,234 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:C 2024-11-12T12:27:07,323 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/eea47d0745df48a99fda6bf97769d060 2024-11-12T12:27:07,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/a1526ad185294ad9a1ff5846f9fd857f is 50, key is test_row_0/B:col10/1731414425910/Put/seqid=0 2024-11-12T12:27:07,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742407_1583 (size=12151) 2024-11-12T12:27:07,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47360 deadline: 1731414487351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,353 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_1' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:27:07,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47346 deadline: 1731414487361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,363 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4a91c05c96a4,38717,1731414257691 for 'test_row_0' in de303568766865fcf42f0cc60107a910 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T12:27:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T12:27:07,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414487421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414487431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414487431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,733 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/a1526ad185294ad9a1ff5846f9fd857f 2024-11-12T12:27:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414487731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/0e48b7a915da41189797025952564979 is 50, key is test_row_0/C:col10/1731414425910/Put/seqid=0 2024-11-12T12:27:07,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742408_1584 (size=12151) 2024-11-12T12:27:07,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414487741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414487742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:07,743 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/0e48b7a915da41189797025952564979 2024-11-12T12:27:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:07,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/eea47d0745df48a99fda6bf97769d060 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/eea47d0745df48a99fda6bf97769d060 2024-11-12T12:27:07,750 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/eea47d0745df48a99fda6bf97769d060, entries=150, sequenceid=234, filesize=30.4 K 2024-11-12T12:27:07,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/a1526ad185294ad9a1ff5846f9fd857f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a1526ad185294ad9a1ff5846f9fd857f 2024-11-12T12:27:07,755 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a1526ad185294ad9a1ff5846f9fd857f, entries=150, sequenceid=234, filesize=11.9 K 2024-11-12T12:27:07,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/0e48b7a915da41189797025952564979 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0e48b7a915da41189797025952564979 2024-11-12T12:27:07,758 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0e48b7a915da41189797025952564979, entries=150, sequenceid=234, filesize=11.9 K 2024-11-12T12:27:07,763 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for de303568766865fcf42f0cc60107a910 in 877ms, sequenceid=234, compaction requested=false 2024-11-12T12:27:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-12T12:27:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-12T12:27:07,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-12T12:27:07,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4890 sec 2024-11-12T12:27:07,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4950 sec 2024-11-12T12:27:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:27:08,244 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T12:27:08,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-12T12:27:08,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d688aa7bb2ae4203940e6df37332d3e4_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414427097/Put/seqid=0 2024-11-12T12:27:08,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742409_1585 (size=14794) 2024-11-12T12:27:08,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414488325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414488329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414488330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T12:27:08,421 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:27:08,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T12:27:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-12T12:27:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T12:27:08,423 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T12:27:08,423 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T12:27:08,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T12:27:08,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414488441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414488441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414488441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T12:27:08,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T12:27:08,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:08,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:08,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:08,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:08,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:08,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414488651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414488651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414488651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,666 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:27:08,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d688aa7bb2ae4203940e6df37332d3e4_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d688aa7bb2ae4203940e6df37332d3e4_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:08,669 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/f5a58b6a973d4acfb4cab58a8c2e0b26, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:08,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/f5a58b6a973d4acfb4cab58a8c2e0b26 is 175, key is test_row_0/A:col10/1731414427097/Put/seqid=0 2024-11-12T12:27:08,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742410_1586 (size=39749) 2024-11-12T12:27:08,673 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/f5a58b6a973d4acfb4cab58a8c2e0b26 2024-11-12T12:27:08,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/557ab06fd4794281bf599148f396d389 is 50, key is test_row_0/B:col10/1731414427097/Put/seqid=0 2024-11-12T12:27:08,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742411_1587 (size=12151) 2024-11-12T12:27:08,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T12:27:08,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:08,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:08,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:08,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:08,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T12:27:08,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T12:27:08,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:08,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:08,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:08,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:08,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:08,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414488961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414488962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414488962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:09,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T12:27:09,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:09,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T12:27:09,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/557ab06fd4794281bf599148f396d389 2024-11-12T12:27:09,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/59dc17e6d42e472ebc8175fb2184ba19 is 50, key is test_row_0/C:col10/1731414427097/Put/seqid=0 2024-11-12T12:27:09,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742412_1588 (size=12151) 2024-11-12T12:27:09,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T12:27:09,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:09,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T12:27:09,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:09,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,472 INFO [Thread-2329 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2329. 2024-11-12T12:27:09,472 DEBUG [Thread-2329 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:09,472 INFO [Thread-2323 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2323. 2024-11-12T12:27:09,472 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,473 DEBUG [Thread-2323 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:09,473 DEBUG [Thread-2323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,473 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:09,473 DEBUG [Thread-2323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:09,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47374 deadline: 1731414489471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:09,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47366 deadline: 1731414489471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:09,474 INFO [Thread-2321 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2321. 2024-11-12T12:27:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:09,474 DEBUG [Thread-2321 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:09,474 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,474 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:09,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:09,475 INFO [Thread-2325 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2325. 2024-11-12T12:27:09,476 DEBUG [Thread-2325 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:09,476 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,476 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:09,476 INFO [Thread-2327 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2327. 2024-11-12T12:27:09,476 DEBUG [Thread-2327 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:09,476 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,476 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:09,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:09,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T12:27:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47368 deadline: 1731414489481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 , the old value is region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T12:27:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., hostname=4a91c05c96a4,38717,1731414257691, seqNum=5 because the exception is null or not the one we care about 2024-11-12T12:27:09,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T12:27:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T12:27:09,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/59dc17e6d42e472ebc8175fb2184ba19 2024-11-12T12:27:09,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/f5a58b6a973d4acfb4cab58a8c2e0b26 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f5a58b6a973d4acfb4cab58a8c2e0b26 2024-11-12T12:27:09,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f5a58b6a973d4acfb4cab58a8c2e0b26, entries=200, sequenceid=255, filesize=38.8 K 2024-11-12T12:27:09,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/557ab06fd4794281bf599148f396d389 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/557ab06fd4794281bf599148f396d389 2024-11-12T12:27:09,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/557ab06fd4794281bf599148f396d389, entries=150, sequenceid=255, filesize=11.9 K 2024-11-12T12:27:09,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/59dc17e6d42e472ebc8175fb2184ba19 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/59dc17e6d42e472ebc8175fb2184ba19 2024-11-12T12:27:09,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/59dc17e6d42e472ebc8175fb2184ba19, entries=150, sequenceid=255, filesize=11.9 K 2024-11-12T12:27:09,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for de303568766865fcf42f0cc60107a910 in 1266ms, sequenceid=255, compaction requested=true 2024-11-12T12:27:09,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:09,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T12:27:09,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:09,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T12:27:09,511 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:09,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:09,511 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:09,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de303568766865fcf42f0cc60107a910:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T12:27:09,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:27:09,511 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:09,511 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:09,512 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/B is initiating minor compaction (all files) 2024-11-12T12:27:09,512 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/A is initiating minor compaction (all files) 2024-11-12T12:27:09,512 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/A in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,512 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/B in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,512 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/43b841e3d5ec4296a7c22aa4a786cebb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a1526ad185294ad9a1ff5846f9fd857f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/557ab06fd4794281bf599148f396d389] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=36.1 K 2024-11-12T12:27:09,512 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0f6748a34b784d30a24521e2141354ee, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/eea47d0745df48a99fda6bf97769d060, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f5a58b6a973d4acfb4cab58a8c2e0b26] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=100.1 K 2024-11-12T12:27:09,512 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,512 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. files: [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0f6748a34b784d30a24521e2141354ee, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/eea47d0745df48a99fda6bf97769d060, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f5a58b6a973d4acfb4cab58a8c2e0b26] 2024-11-12T12:27:09,512 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 43b841e3d5ec4296a7c22aa4a786cebb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414425237 2024-11-12T12:27:09,512 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f6748a34b784d30a24521e2141354ee, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414425237 2024-11-12T12:27:09,512 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting a1526ad185294ad9a1ff5846f9fd857f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731414425900 2024-11-12T12:27:09,512 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea47d0745df48a99fda6bf97769d060, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731414425900 2024-11-12T12:27:09,512 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 557ab06fd4794281bf599148f396d389, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731414427097 2024-11-12T12:27:09,513 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a58b6a973d4acfb4cab58a8c2e0b26, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731414427089 2024-11-12T12:27:09,517 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:09,518 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:09,518 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/502e2841bc184a86a80dab5eea3610c3 is 50, key is test_row_0/B:col10/1731414427097/Put/seqid=0 2024-11-12T12:27:09,518 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112090db67d1c3e462b982c64be9d453953_de303568766865fcf42f0cc60107a910 store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:09,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742413_1589 (size=12765) 2024-11-12T12:27:09,524 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112090db67d1c3e462b982c64be9d453953_de303568766865fcf42f0cc60107a910, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:09,524 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112090db67d1c3e462b982c64be9d453953_de303568766865fcf42f0cc60107a910 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:09,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742414_1590 (size=4469) 2024-11-12T12:27:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T12:27:09,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T12:27:09,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,645 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-12T12:27:09,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112400b0a651c544d0dbb16a1e44cbcd895_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414428305/Put/seqid=0 2024-11-12T12:27:09,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742415_1591 (size=12454) 2024-11-12T12:27:09,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:27:09,662 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112400b0a651c544d0dbb16a1e44cbcd895_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112400b0a651c544d0dbb16a1e44cbcd895_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:09,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/b5bdcfe8cb8140a8a0074f68898de186, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:09,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/b5bdcfe8cb8140a8a0074f68898de186 is 175, key is test_row_0/A:col10/1731414428305/Put/seqid=0 2024-11-12T12:27:09,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742416_1592 (size=31255) 2024-11-12T12:27:09,924 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/502e2841bc184a86a80dab5eea3610c3 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/502e2841bc184a86a80dab5eea3610c3 2024-11-12T12:27:09,926 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/B of de303568766865fcf42f0cc60107a910 into 502e2841bc184a86a80dab5eea3610c3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:09,926 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:09,926 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/B, priority=13, startTime=1731414429511; duration=0sec 2024-11-12T12:27:09,927 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T12:27:09,927 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:B 2024-11-12T12:27:09,927 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T12:27:09,927 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T12:27:09,927 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1541): de303568766865fcf42f0cc60107a910/C is initiating minor compaction (all files) 2024-11-12T12:27:09,927 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de303568766865fcf42f0cc60107a910/C in TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:09,927 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/31bbf446f0ab4952bb0ae581435e350d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0e48b7a915da41189797025952564979, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/59dc17e6d42e472ebc8175fb2184ba19] into tmpdir=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp, totalSize=36.1 K 2024-11-12T12:27:09,927 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 31bbf446f0ab4952bb0ae581435e350d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731414425237 2024-11-12T12:27:09,928 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e48b7a915da41189797025952564979, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731414425900 2024-11-12T12:27:09,928 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] compactions.Compactor(225): Compacting 59dc17e6d42e472ebc8175fb2184ba19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731414427097 2024-11-12T12:27:09,930 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:09,930 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/f0e90c1cdb1b4ac5935187c86e3a0a12 is 175, key is test_row_0/A:col10/1731414427097/Put/seqid=0 2024-11-12T12:27:09,933 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de303568766865fcf42f0cc60107a910#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T12:27:09,933 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/7ae2a6165a3641cb8c3e97599f0e2cfc is 50, key is test_row_0/C:col10/1731414427097/Put/seqid=0 2024-11-12T12:27:09,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742417_1593 (size=31719) 2024-11-12T12:27:09,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742418_1594 (size=12765) 2024-11-12T12:27:10,066 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/b5bdcfe8cb8140a8a0074f68898de186 2024-11-12T12:27:10,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/d0b002a9b608481281a46fd230b2cb96 is 50, key is test_row_0/B:col10/1731414428305/Put/seqid=0 2024-11-12T12:27:10,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742419_1595 (size=12301) 2024-11-12T12:27:10,340 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/f0e90c1cdb1b4ac5935187c86e3a0a12 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f0e90c1cdb1b4ac5935187c86e3a0a12 2024-11-12T12:27:10,341 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/7ae2a6165a3641cb8c3e97599f0e2cfc as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7ae2a6165a3641cb8c3e97599f0e2cfc 2024-11-12T12:27:10,343 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/A of de303568766865fcf42f0cc60107a910 into f0e90c1cdb1b4ac5935187c86e3a0a12(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:10,343 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:10,343 INFO [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/A, priority=13, startTime=1731414429511; duration=0sec 2024-11-12T12:27:10,343 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:10,343 DEBUG [RS:0;4a91c05c96a4:38717-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:A 2024-11-12T12:27:10,344 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de303568766865fcf42f0cc60107a910/C of de303568766865fcf42f0cc60107a910 into 7ae2a6165a3641cb8c3e97599f0e2cfc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T12:27:10,344 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:10,344 INFO [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., storeName=de303568766865fcf42f0cc60107a910/C, priority=13, startTime=1731414429511; duration=0sec 2024-11-12T12:27:10,344 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T12:27:10,344 DEBUG [RS:0;4a91c05c96a4:38717-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de303568766865fcf42f0cc60107a910:C 2024-11-12T12:27:10,475 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/d0b002a9b608481281a46fd230b2cb96 2024-11-12T12:27:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/0cd0f6a0c64f404e9057cf1ad637e4a5 is 50, key is test_row_0/C:col10/1731414428305/Put/seqid=0 2024-11-12T12:27:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] regionserver.HRegion(8855): Flush requested on de303568766865fcf42f0cc60107a910 2024-11-12T12:27:10,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. as already flushing 2024-11-12T12:27:10,482 INFO [Thread-2316 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2316. 2024-11-12T12:27:10,482 DEBUG [Thread-2316 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:10,482 DEBUG [Thread-2316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:10,483 DEBUG [Thread-2316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:10,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:10,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742420_1596 (size=12301) 2024-11-12T12:27:10,493 INFO [Thread-2314 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2314. 2024-11-12T12:27:10,493 DEBUG [Thread-2314 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:10,493 DEBUG [Thread-2314 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:10,493 DEBUG [Thread-2314 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:10,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:10,502 INFO [Thread-2318 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2318. 2024-11-12T12:27:10,502 DEBUG [Thread-2318 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:10,502 DEBUG [Thread-2318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:10,502 DEBUG [Thread-2318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:10,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T12:27:10,885 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/0cd0f6a0c64f404e9057cf1ad637e4a5 2024-11-12T12:27:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/b5bdcfe8cb8140a8a0074f68898de186 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/b5bdcfe8cb8140a8a0074f68898de186 2024-11-12T12:27:10,891 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/b5bdcfe8cb8140a8a0074f68898de186, entries=150, sequenceid=273, filesize=30.5 K 2024-11-12T12:27:10,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/d0b002a9b608481281a46fd230b2cb96 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d0b002a9b608481281a46fd230b2cb96 2024-11-12T12:27:10,894 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d0b002a9b608481281a46fd230b2cb96, entries=150, sequenceid=273, filesize=12.0 K 2024-11-12T12:27:10,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/0cd0f6a0c64f404e9057cf1ad637e4a5 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0cd0f6a0c64f404e9057cf1ad637e4a5 2024-11-12T12:27:10,896 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0cd0f6a0c64f404e9057cf1ad637e4a5, entries=150, sequenceid=273, filesize=12.0 K 2024-11-12T12:27:10,897 INFO [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=20.13 KB/20610 for de303568766865fcf42f0cc60107a910 in 1252ms, sequenceid=273, compaction requested=false 2024-11-12T12:27:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for de303568766865fcf42f0cc60107a910: 2024-11-12T12:27:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4a91c05c96a4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-12T12:27:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-12T12:27:10,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-12T12:27:10,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4750 sec 2024-11-12T12:27:10,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.4790 sec 2024-11-12T12:27:11,372 INFO [Thread-2310 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2310. 2024-11-12T12:27:11,372 DEBUG [Thread-2310 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:11,372 INFO [Thread-2312 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2312. 2024-11-12T12:27:11,372 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:11,372 DEBUG [Thread-2312 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T12:27:11,373 DEBUG [Thread-2312 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:11,373 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:11,373 DEBUG [Thread-2312 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:11,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:11,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T12:27:12,592 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4304 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4230 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4194 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4316 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4222 2024-11-12T12:27:12,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T12:27:12,592 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T12:27:12,592 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T12:27:12,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:12,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:12,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:12,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T12:27:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T12:27:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T12:27:12,596 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414432595"}]},"ts":"1731414432595"} 2024-11-12T12:27:12,596 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T12:27:12,596 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T12:27:12,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T12:27:12,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, UNASSIGN}] 2024-11-12T12:27:12,598 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, UNASSIGN 2024-11-12T12:27:12,599 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=de303568766865fcf42f0cc60107a910, regionState=CLOSING, regionLocation=4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:12,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, UNASSIGN because future has completed 2024-11-12T12:27:12,600 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T12:27:12,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691}] 2024-11-12T12:27:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T12:27:12,752 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close de303568766865fcf42f0cc60107a910 2024-11-12T12:27:12,752 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T12:27:12,752 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing de303568766865fcf42f0cc60107a910, disabling compactions & flushes 2024-11-12T12:27:12,752 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:12,752 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:12,752 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. after waiting 0 ms 2024-11-12T12:27:12,752 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:12,752 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing de303568766865fcf42f0cc60107a910 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-12T12:27:12,756 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111266f04fe9224d4e76a41ab586f4f4d30f_de303568766865fcf42f0cc60107a910 is 50, key is test_row_0/A:col10/1731414431371/Put/seqid=0 2024-11-12T12:27:12,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742421_1597 (size=9914) 2024-11-12T12:27:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T12:27:13,159 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T12:27:13,162 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111266f04fe9224d4e76a41ab586f4f4d30f_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111266f04fe9224d4e76a41ab586f4f4d30f_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:13,162 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/18374624ef3b46b79e6fdb03cdf803aa, store: [table=TestAcidGuarantees family=A region=de303568766865fcf42f0cc60107a910] 2024-11-12T12:27:13,163 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/18374624ef3b46b79e6fdb03cdf803aa is 175, key is test_row_0/A:col10/1731414431371/Put/seqid=0 2024-11-12T12:27:13,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742422_1598 (size=22561) 2024-11-12T12:27:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T12:27:13,566 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/18374624ef3b46b79e6fdb03cdf803aa 2024-11-12T12:27:13,570 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/17c4a11563764f478e7f5f0666ed7a8f is 50, key is test_row_0/B:col10/1731414431371/Put/seqid=0 2024-11-12T12:27:13,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742423_1599 (size=9857) 2024-11-12T12:27:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T12:27:13,974 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/17c4a11563764f478e7f5f0666ed7a8f 2024-11-12T12:27:13,978 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/ab644d48195a45efaa13c06aa9052935 is 50, key is test_row_0/C:col10/1731414431371/Put/seqid=0 2024-11-12T12:27:13,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742424_1600 (size=9857) 2024-11-12T12:27:14,381 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/ab644d48195a45efaa13c06aa9052935 2024-11-12T12:27:14,383 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/A/18374624ef3b46b79e6fdb03cdf803aa as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/18374624ef3b46b79e6fdb03cdf803aa 2024-11-12T12:27:14,386 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/18374624ef3b46b79e6fdb03cdf803aa, entries=100, sequenceid=284, filesize=22.0 K 2024-11-12T12:27:14,387 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/B/17c4a11563764f478e7f5f0666ed7a8f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/17c4a11563764f478e7f5f0666ed7a8f 2024-11-12T12:27:14,389 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/17c4a11563764f478e7f5f0666ed7a8f, entries=100, sequenceid=284, filesize=9.6 K 2024-11-12T12:27:14,390 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/.tmp/C/ab644d48195a45efaa13c06aa9052935 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/ab644d48195a45efaa13c06aa9052935 2024-11-12T12:27:14,392 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/ab644d48195a45efaa13c06aa9052935, entries=100, sequenceid=284, filesize=9.6 K 2024-11-12T12:27:14,393 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for de303568766865fcf42f0cc60107a910 in 1640ms, sequenceid=284, compaction requested=true 2024-11-12T12:27:14,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/3ea8af1d8a874a6a869bc9dd85761d1a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/466f9beceb34492ebad4bb3f3446f8f5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f33c3fe469774293849b335e9cef2006, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/229554eba65c4a6888418b6594105972, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/aad29d0ae1014d85bb998ae127b604ac, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/cfebd976906245e7b512b80add5af0a5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0047aa77dc394e30b1d6f52d9dfdd7b5, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/8bc54731096d4567a5e34216f6889db6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/22b22cdf853a4ed398ce23148ee17ee0, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/568a4e80123d4baba6f430bccb0e733f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/bd463f724ff04f35bf493a6129b94470, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d07627e641cd4c278d391cec18d4d50c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/5d01163452ca41f083acb934d426598f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d22d4eb1769944989e37257549d752ef, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d343a53b791942f5b24b01af9b75f855, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0f6748a34b784d30a24521e2141354ee, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/eea47d0745df48a99fda6bf97769d060, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f5a58b6a973d4acfb4cab58a8c2e0b26] to archive 2024-11-12T12:27:14,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:27:14,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/3ea8af1d8a874a6a869bc9dd85761d1a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/3ea8af1d8a874a6a869bc9dd85761d1a 2024-11-12T12:27:14,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/466f9beceb34492ebad4bb3f3446f8f5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/466f9beceb34492ebad4bb3f3446f8f5 2024-11-12T12:27:14,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f33c3fe469774293849b335e9cef2006 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f33c3fe469774293849b335e9cef2006 2024-11-12T12:27:14,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/229554eba65c4a6888418b6594105972 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/229554eba65c4a6888418b6594105972 2024-11-12T12:27:14,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/aad29d0ae1014d85bb998ae127b604ac to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/aad29d0ae1014d85bb998ae127b604ac 2024-11-12T12:27:14,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/cfebd976906245e7b512b80add5af0a5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/cfebd976906245e7b512b80add5af0a5 2024-11-12T12:27:14,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0047aa77dc394e30b1d6f52d9dfdd7b5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0047aa77dc394e30b1d6f52d9dfdd7b5 2024-11-12T12:27:14,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/8bc54731096d4567a5e34216f6889db6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/8bc54731096d4567a5e34216f6889db6 2024-11-12T12:27:14,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/22b22cdf853a4ed398ce23148ee17ee0 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/22b22cdf853a4ed398ce23148ee17ee0 2024-11-12T12:27:14,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/568a4e80123d4baba6f430bccb0e733f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/568a4e80123d4baba6f430bccb0e733f 2024-11-12T12:27:14,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/bd463f724ff04f35bf493a6129b94470 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/bd463f724ff04f35bf493a6129b94470 2024-11-12T12:27:14,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d07627e641cd4c278d391cec18d4d50c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d07627e641cd4c278d391cec18d4d50c 2024-11-12T12:27:14,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/5d01163452ca41f083acb934d426598f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/5d01163452ca41f083acb934d426598f 2024-11-12T12:27:14,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d22d4eb1769944989e37257549d752ef to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d22d4eb1769944989e37257549d752ef 2024-11-12T12:27:14,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d343a53b791942f5b24b01af9b75f855 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/d343a53b791942f5b24b01af9b75f855 2024-11-12T12:27:14,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0f6748a34b784d30a24521e2141354ee to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/0f6748a34b784d30a24521e2141354ee 2024-11-12T12:27:14,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/eea47d0745df48a99fda6bf97769d060 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/eea47d0745df48a99fda6bf97769d060 2024-11-12T12:27:14,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f5a58b6a973d4acfb4cab58a8c2e0b26 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f5a58b6a973d4acfb4cab58a8c2e0b26 2024-11-12T12:27:14,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/e420809df4814c069750bace5418edcc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/9d4000935c4e4fe6b8cfab1abcc01301, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6984b1eb67d849ed8be0582d895da3b7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/095a02c8359240599d56f27014e345f7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a74a0651e39d44eda1654ca3bdeac904, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/697a18c4886d40efbf06d3c901debb4c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1db140d1a75245969d64e2600b8a0346, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6b92b3b0d5b843e49dd49194cf25dd6d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/39a7bed0bbc849c1842dfc37b0cd8ca2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1c49575f8fe745d4b26f259a25c49f3c, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d3ef135c8a5941d8b52d018efa9b97cc, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/33d1b10d46564f09893ed5c1d74747f2, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/bdc882e71897432292537fadf0758679, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/2bf0a3966a7a43f387388d1b4a0a9c0a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/43b841e3d5ec4296a7c22aa4a786cebb, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/25cf2df45b4548ac8c3e44ec01512a81, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a1526ad185294ad9a1ff5846f9fd857f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/557ab06fd4794281bf599148f396d389] to archive 2024-11-12T12:27:14,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:27:14,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/e420809df4814c069750bace5418edcc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/e420809df4814c069750bace5418edcc 2024-11-12T12:27:14,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/9d4000935c4e4fe6b8cfab1abcc01301 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/9d4000935c4e4fe6b8cfab1abcc01301 2024-11-12T12:27:14,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6984b1eb67d849ed8be0582d895da3b7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6984b1eb67d849ed8be0582d895da3b7 2024-11-12T12:27:14,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/095a02c8359240599d56f27014e345f7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/095a02c8359240599d56f27014e345f7 2024-11-12T12:27:14,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a74a0651e39d44eda1654ca3bdeac904 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a74a0651e39d44eda1654ca3bdeac904 2024-11-12T12:27:14,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/697a18c4886d40efbf06d3c901debb4c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/697a18c4886d40efbf06d3c901debb4c 2024-11-12T12:27:14,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1db140d1a75245969d64e2600b8a0346 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1db140d1a75245969d64e2600b8a0346 2024-11-12T12:27:14,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6b92b3b0d5b843e49dd49194cf25dd6d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/6b92b3b0d5b843e49dd49194cf25dd6d 2024-11-12T12:27:14,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/39a7bed0bbc849c1842dfc37b0cd8ca2 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/39a7bed0bbc849c1842dfc37b0cd8ca2 2024-11-12T12:27:14,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1c49575f8fe745d4b26f259a25c49f3c to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/1c49575f8fe745d4b26f259a25c49f3c 2024-11-12T12:27:14,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d3ef135c8a5941d8b52d018efa9b97cc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d3ef135c8a5941d8b52d018efa9b97cc 2024-11-12T12:27:14,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/33d1b10d46564f09893ed5c1d74747f2 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/33d1b10d46564f09893ed5c1d74747f2 2024-11-12T12:27:14,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/bdc882e71897432292537fadf0758679 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/bdc882e71897432292537fadf0758679 2024-11-12T12:27:14,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/2bf0a3966a7a43f387388d1b4a0a9c0a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/2bf0a3966a7a43f387388d1b4a0a9c0a 2024-11-12T12:27:14,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/43b841e3d5ec4296a7c22aa4a786cebb to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/43b841e3d5ec4296a7c22aa4a786cebb 2024-11-12T12:27:14,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/25cf2df45b4548ac8c3e44ec01512a81 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/25cf2df45b4548ac8c3e44ec01512a81 2024-11-12T12:27:14,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a1526ad185294ad9a1ff5846f9fd857f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/a1526ad185294ad9a1ff5846f9fd857f 2024-11-12T12:27:14,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/557ab06fd4794281bf599148f396d389 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/557ab06fd4794281bf599148f396d389 2024-11-12T12:27:14,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b88c7deb244f4f16a4c1a79e412c04ac, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/dae957eef59d43b6971e3ba09ad1c8ab, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/676a61cc0a8c4afdbfc16ef0bc7b201a, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/1cdf6df69bee401a93c1d58d44b488a6, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7328bcb4515e490e98010dd8b99c7ae7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/d6af290ae7e5408a9888c37e91ae2de8, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7c6512c1a3264ccd8cead5976e11cd1d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/c788818d611e41409a25f72de6fe8e15, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e62183bba944282b04f8af4990aaf52, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b7c9d4aae2aa4caa8d4427b47a907d8f, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/772bbd4724694f7baa08690fe7d2d2ff, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/4bf502fbb04e4763a851b466af0acf57, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e7e7588e99c43609081d80fb8b7beb3, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/70d7801d86654c9b850f54e2334598c7, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/31bbf446f0ab4952bb0ae581435e350d, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/9bb38b016fae4791aa3be72d820d3e27, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0e48b7a915da41189797025952564979, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/59dc17e6d42e472ebc8175fb2184ba19] to archive 2024-11-12T12:27:14,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T12:27:14,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b88c7deb244f4f16a4c1a79e412c04ac to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b88c7deb244f4f16a4c1a79e412c04ac 2024-11-12T12:27:14,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/dae957eef59d43b6971e3ba09ad1c8ab to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/dae957eef59d43b6971e3ba09ad1c8ab 2024-11-12T12:27:14,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/676a61cc0a8c4afdbfc16ef0bc7b201a to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/676a61cc0a8c4afdbfc16ef0bc7b201a 2024-11-12T12:27:14,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/1cdf6df69bee401a93c1d58d44b488a6 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/1cdf6df69bee401a93c1d58d44b488a6 2024-11-12T12:27:14,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7328bcb4515e490e98010dd8b99c7ae7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7328bcb4515e490e98010dd8b99c7ae7 2024-11-12T12:27:14,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/d6af290ae7e5408a9888c37e91ae2de8 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/d6af290ae7e5408a9888c37e91ae2de8 2024-11-12T12:27:14,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7c6512c1a3264ccd8cead5976e11cd1d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7c6512c1a3264ccd8cead5976e11cd1d 2024-11-12T12:27:14,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/c788818d611e41409a25f72de6fe8e15 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/c788818d611e41409a25f72de6fe8e15 2024-11-12T12:27:14,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e62183bba944282b04f8af4990aaf52 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e62183bba944282b04f8af4990aaf52 2024-11-12T12:27:14,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b7c9d4aae2aa4caa8d4427b47a907d8f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/b7c9d4aae2aa4caa8d4427b47a907d8f 2024-11-12T12:27:14,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/772bbd4724694f7baa08690fe7d2d2ff to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/772bbd4724694f7baa08690fe7d2d2ff 2024-11-12T12:27:14,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/4bf502fbb04e4763a851b466af0acf57 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/4bf502fbb04e4763a851b466af0acf57 2024-11-12T12:27:14,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e7e7588e99c43609081d80fb8b7beb3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/6e7e7588e99c43609081d80fb8b7beb3 2024-11-12T12:27:14,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/70d7801d86654c9b850f54e2334598c7 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/70d7801d86654c9b850f54e2334598c7 2024-11-12T12:27:14,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/31bbf446f0ab4952bb0ae581435e350d to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/31bbf446f0ab4952bb0ae581435e350d 2024-11-12T12:27:14,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/9bb38b016fae4791aa3be72d820d3e27 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/9bb38b016fae4791aa3be72d820d3e27 2024-11-12T12:27:14,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0e48b7a915da41189797025952564979 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0e48b7a915da41189797025952564979 2024-11-12T12:27:14,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/59dc17e6d42e472ebc8175fb2184ba19 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/59dc17e6d42e472ebc8175fb2184ba19 2024-11-12T12:27:14,439 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/recovered.edits/287.seqid, newMaxSeqId=287, maxSeqId=4 2024-11-12T12:27:14,439 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. 2024-11-12T12:27:14,439 DEBUG [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for de303568766865fcf42f0cc60107a910: Waiting for close lock at 1731414432752Running coprocessor pre-close hooks at 1731414432752Disabling compacts and flushes for region at 1731414432752Disabling writes for close at 1731414432752Obtaining lock to block concurrent updates at 1731414432752Preparing flush snapshotting stores in de303568766865fcf42f0cc60107a910 at 1731414432752Finished memstore snapshotting TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731414432752Flushing stores of TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910. at 1731414432753 (+1 ms)Flushing de303568766865fcf42f0cc60107a910/A: creating writer at 1731414432753Flushing mob file de303568766865fcf42f0cc60107a910/A: appending metadata at 1731414432756 (+3 ms)Flushing mob file de303568766865fcf42f0cc60107a910/A: closing flushed file at 1731414432756Flushing de303568766865fcf42f0cc60107a910/A: appending metadata at 1731414433162 (+406 ms)Flushing de303568766865fcf42f0cc60107a910/A: closing flushed file at 1731414433162Flushing de303568766865fcf42f0cc60107a910/B: creating writer at 1731414433568 (+406 ms)Flushing de303568766865fcf42f0cc60107a910/B: appending metadata at 1731414433570 (+2 ms)Flushing de303568766865fcf42f0cc60107a910/B: closing flushed file at 1731414433570Flushing de303568766865fcf42f0cc60107a910/C: creating writer at 1731414433976 (+406 ms)Flushing de303568766865fcf42f0cc60107a910/C: appending metadata at 1731414433978 (+2 ms)Flushing de303568766865fcf42f0cc60107a910/C: closing flushed file at 1731414433978Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46542107: reopening flushed file at 1731414434383 (+405 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@115ed561: reopening flushed file at 1731414434386 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39164b23: reopening flushed file at 1731414434389 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for de303568766865fcf42f0cc60107a910 in 1640ms, sequenceid=284, compaction requested=true at 1731414434393 (+4 ms)Writing region close event to WAL at 1731414434436 (+43 ms)Running coprocessor post-close hooks at 1731414434439 (+3 ms)Closed at 1731414434439 2024-11-12T12:27:14,441 INFO [RS_CLOSE_REGION-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,441 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=de303568766865fcf42f0cc60107a910, regionState=CLOSED 2024-11-12T12:27:14,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 because future has completed 2024-11-12T12:27:14,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-12T12:27:14,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure de303568766865fcf42f0cc60107a910, server=4a91c05c96a4,38717,1731414257691 in 1.8430 sec 2024-11-12T12:27:14,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-12T12:27:14,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de303568766865fcf42f0cc60107a910, UNASSIGN in 1.8460 sec 2024-11-12T12:27:14,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-12T12:27:14,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8480 sec 2024-11-12T12:27:14,447 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731414434447"}]},"ts":"1731414434447"} 2024-11-12T12:27:14,448 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T12:27:14,448 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T12:27:14,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8560 sec 2024-11-12T12:27:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T12:27:14,741 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:27:14,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T12:27:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:27:14,743 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:27:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T12:27:14,743 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:27:14,745 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,747 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C, FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/recovered.edits] 2024-11-12T12:27:14,748 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/18374624ef3b46b79e6fdb03cdf803aa to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/18374624ef3b46b79e6fdb03cdf803aa 2024-11-12T12:27:14,749 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/b5bdcfe8cb8140a8a0074f68898de186 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/b5bdcfe8cb8140a8a0074f68898de186 2024-11-12T12:27:14,750 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f0e90c1cdb1b4ac5935187c86e3a0a12 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/A/f0e90c1cdb1b4ac5935187c86e3a0a12 2024-11-12T12:27:14,751 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/17c4a11563764f478e7f5f0666ed7a8f to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/17c4a11563764f478e7f5f0666ed7a8f 2024-11-12T12:27:14,752 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/502e2841bc184a86a80dab5eea3610c3 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/502e2841bc184a86a80dab5eea3610c3 2024-11-12T12:27:14,753 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d0b002a9b608481281a46fd230b2cb96 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/B/d0b002a9b608481281a46fd230b2cb96 2024-11-12T12:27:14,754 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0cd0f6a0c64f404e9057cf1ad637e4a5 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/0cd0f6a0c64f404e9057cf1ad637e4a5 2024-11-12T12:27:14,754 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7ae2a6165a3641cb8c3e97599f0e2cfc to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/7ae2a6165a3641cb8c3e97599f0e2cfc 2024-11-12T12:27:14,755 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/ab644d48195a45efaa13c06aa9052935 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/C/ab644d48195a45efaa13c06aa9052935 2024-11-12T12:27:14,756 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/recovered.edits/287.seqid to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910/recovered.edits/287.seqid 2024-11-12T12:27:14,757 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/default/TestAcidGuarantees/de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,757 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T12:27:14,757 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T12:27:14,757 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T12:27:14,759 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120e06b986462449278b96b6fd5a10f37c_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120e06b986462449278b96b6fd5a10f37c_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,760 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121e2d246a17c34f8ca3b95210434f2c87_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121e2d246a17c34f8ca3b95210434f2c87_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,760 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122c4dc999b8df442aa01c24acbfac6bc8_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122c4dc999b8df442aa01c24acbfac6bc8_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,761 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111235a73eed52814e0d8d2f1416fcb23af1_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111235a73eed52814e0d8d2f1416fcb23af1_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,762 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123e91c55352a34198b6abea67876e9962_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123e91c55352a34198b6abea67876e9962_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,763 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112400b0a651c544d0dbb16a1e44cbcd895_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112400b0a651c544d0dbb16a1e44cbcd895_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,763 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111258f824404e01423d9d0a433b52cd10a7_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111258f824404e01423d9d0a433b52cd10a7_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,764 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125c10c4d66c18411ea5fddf66fcf53673_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125c10c4d66c18411ea5fddf66fcf53673_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,764 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112634e163bf0014ca194f8f4bcaae81107_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112634e163bf0014ca194f8f4bcaae81107_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,765 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111266f04fe9224d4e76a41ab586f4f4d30f_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111266f04fe9224d4e76a41ab586f4f4d30f_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,766 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126e866b3586a4424bbd0638d400c0fe9f_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126e866b3586a4424bbd0638d400c0fe9f_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,766 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bc010c524119467d998f4479de9f3f0d_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bc010c524119467d998f4479de9f3f0d_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,767 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d2134d136dd6495ca9a7f300e441bec4_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d2134d136dd6495ca9a7f300e441bec4_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,768 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d57025c44fc04537b1a03ed420dbb5d5_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d57025c44fc04537b1a03ed420dbb5d5_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,768 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d688aa7bb2ae4203940e6df37332d3e4_de303568766865fcf42f0cc60107a910 to hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d688aa7bb2ae4203940e6df37332d3e4_de303568766865fcf42f0cc60107a910 2024-11-12T12:27:14,769 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T12:27:14,770 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:27:14,772 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T12:27:14,773 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T12:27:14,774 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:27:14,774 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T12:27:14,774 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731414434774"}]},"ts":"9223372036854775807"} 2024-11-12T12:27:14,776 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T12:27:14,776 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => de303568766865fcf42f0cc60107a910, NAME => 'TestAcidGuarantees,,1731414406343.de303568766865fcf42f0cc60107a910.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T12:27:14,776 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T12:27:14,776 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731414434776"}]},"ts":"9223372036854775807"} 2024-11-12T12:27:14,777 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T12:27:14,778 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T12:27:14,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-11-12T12:27:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41939 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T12:27:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T12:27:14,851 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T12:27:14,860 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 240), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=454 (was 476), ProcessCount=11 (was 11), AvailableMemoryMB=6660 (was 6712) 2024-11-12T12:27:14,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T12:27:14,860 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T12:27:14,860 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T12:27:14,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:14,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:14,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T12:27:14,860 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T12:27:14,860 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1866486918, stopped=false 2024-11-12T12:27:14,861 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4a91c05c96a4,41939,1731414256942 2024-11-12T12:27:14,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T12:27:14,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T12:27:14,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:27:14,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:27:14,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T12:27:14,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T12:27:14,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T12:27:14,920 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T12:27:14,920 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T12:27:14,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:14,921 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4a91c05c96a4,38717,1731414257691' ***** 2024-11-12T12:27:14,921 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T12:27:14,921 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T12:27:14,921 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T12:27:14,921 INFO [RS:0;4a91c05c96a4:38717 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T12:27:14,921 INFO [RS:0;4a91c05c96a4:38717 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T12:27:14,921 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(959): stopping server 4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:14,921 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T12:27:14,921 INFO [RS:0;4a91c05c96a4:38717 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4a91c05c96a4:38717. 2024-11-12T12:27:14,922 DEBUG [RS:0;4a91c05c96a4:38717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T12:27:14,922 DEBUG [RS:0;4a91c05c96a4:38717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:14,922 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T12:27:14,922 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T12:27:14,922 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T12:27:14,922 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T12:27:14,922 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T12:27:14,922 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T12:27:14,923 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T12:27:14,923 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T12:27:14,923 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T12:27:14,923 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T12:27:14,923 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T12:27:14,923 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T12:27:14,923 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-12T12:27:14,943 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/info/9d50b73fb2b44179b578e5c5ad605b18 is 95, key is TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827./info:/1731414288483/DeleteFamily/seqid=0 2024-11-12T12:27:14,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742425_1601 (size=6514) 2024-11-12T12:27:14,951 INFO [regionserver/4a91c05c96a4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T12:27:15,123 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T12:27:15,323 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T12:27:15,350 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/info/9d50b73fb2b44179b578e5c5ad605b18 2024-11-12T12:27:15,367 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/ns/6a0cefbe29de4087a664ab0e58780cbf is 93, key is TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827./ns:/1731414288442/DeleteFamily/seqid=0 2024-11-12T12:27:15,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742426_1602 (size=5906) 2024-11-12T12:27:15,523 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T12:27:15,723 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T12:27:15,770 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/ns/6a0cefbe29de4087a664ab0e58780cbf 2024-11-12T12:27:15,787 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/rep_barrier/92baf93ef20146e580c3f25949b8121d is 102, key is TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827./rep_barrier:/1731414288442/DeleteFamily/seqid=0 2024-11-12T12:27:15,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742427_1603 (size=6025) 2024-11-12T12:27:15,836 INFO [regionserver/4a91c05c96a4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T12:27:15,836 INFO [regionserver/4a91c05c96a4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T12:27:15,924 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T12:27:16,047 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T12:27:16,124 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T12:27:16,191 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/rep_barrier/92baf93ef20146e580c3f25949b8121d 2024-11-12T12:27:16,206 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/table/b27e1ce05d154fdb89217494802dd446 is 96, key is TestAcidGuarantees,,1731414262251.03699d6939471fdc6521cd2a2d843827./table:/1731414288442/DeleteFamily/seqid=0 2024-11-12T12:27:16,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742428_1604 (size=5936) 2024-11-12T12:27:16,324 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T12:27:16,524 DEBUG [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T12:27:16,610 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/table/b27e1ce05d154fdb89217494802dd446 2024-11-12T12:27:16,612 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/info/9d50b73fb2b44179b578e5c5ad605b18 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/info/9d50b73fb2b44179b578e5c5ad605b18 2024-11-12T12:27:16,615 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/info/9d50b73fb2b44179b578e5c5ad605b18, entries=12, sequenceid=90, filesize=6.4 K 2024-11-12T12:27:16,615 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/ns/6a0cefbe29de4087a664ab0e58780cbf as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/ns/6a0cefbe29de4087a664ab0e58780cbf 2024-11-12T12:27:16,618 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/ns/6a0cefbe29de4087a664ab0e58780cbf, entries=8, sequenceid=90, filesize=5.8 K 2024-11-12T12:27:16,618 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/rep_barrier/92baf93ef20146e580c3f25949b8121d as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/rep_barrier/92baf93ef20146e580c3f25949b8121d 2024-11-12T12:27:16,621 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/rep_barrier/92baf93ef20146e580c3f25949b8121d, entries=6, sequenceid=90, filesize=5.9 K 2024-11-12T12:27:16,621 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/.tmp/table/b27e1ce05d154fdb89217494802dd446 as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/table/b27e1ce05d154fdb89217494802dd446 2024-11-12T12:27:16,624 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/table/b27e1ce05d154fdb89217494802dd446, entries=7, sequenceid=90, filesize=5.8 K 2024-11-12T12:27:16,625 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1701ms, sequenceid=90, compaction requested=false 2024-11-12T12:27:16,628 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-12T12:27:16,628 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T12:27:16,628 INFO [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T12:27:16,628 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731414434922Running coprocessor pre-close hooks at 1731414434923 (+1 ms)Disabling compacts and flushes for region at 1731414434923Disabling writes for close at 1731414434923Obtaining lock to block concurrent updates at 1731414434923Preparing flush snapshotting stores in 1588230740 at 1731414434923Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731414434923Flushing stores of hbase:meta,,1.1588230740 at 1731414434924 (+1 ms)Flushing 1588230740/info: creating writer at 1731414434924Flushing 1588230740/info: appending metadata at 1731414434943 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731414434943Flushing 1588230740/ns: creating writer at 1731414435353 (+410 ms)Flushing 1588230740/ns: appending metadata at 1731414435367 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731414435367Flushing 1588230740/rep_barrier: creating writer at 1731414435773 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731414435787 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731414435787Flushing 1588230740/table: creating writer at 1731414436193 (+406 ms)Flushing 1588230740/table: appending metadata at 1731414436206 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731414436206Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fa82ca4: reopening flushed file at 1731414436612 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35ddba06: reopening flushed file at 1731414436615 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ab2a94f: reopening flushed file at 1731414436618 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@328c0614: reopening flushed file at 1731414436621 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1701ms, sequenceid=90, compaction requested=false at 1731414436625 (+4 ms)Writing region close event to WAL at 1731414436626 (+1 ms)Running coprocessor post-close hooks at 1731414436628 (+2 ms)Closed at 1731414436628 2024-11-12T12:27:16,628 DEBUG [RS_CLOSE_META-regionserver/4a91c05c96a4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T12:27:16,724 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(976): stopping server 4a91c05c96a4,38717,1731414257691; all regions closed. 2024-11-12T12:27:16,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741834_1010 (size=25344) 2024-11-12T12:27:16,730 DEBUG [RS:0;4a91c05c96a4:38717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/oldWALs 2024-11-12T12:27:16,730 INFO [RS:0;4a91c05c96a4:38717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4a91c05c96a4%2C38717%2C1731414257691.meta:.meta(num 1731414261572) 2024-11-12T12:27:16,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741833_1009 (size=13651292) 2024-11-12T12:27:16,733 DEBUG [RS:0;4a91c05c96a4:38717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/oldWALs 2024-11-12T12:27:16,733 INFO [RS:0;4a91c05c96a4:38717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4a91c05c96a4%2C38717%2C1731414257691:(num 1731414260940) 2024-11-12T12:27:16,733 DEBUG [RS:0;4a91c05c96a4:38717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T12:27:16,733 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T12:27:16,733 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T12:27:16,733 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.ChoreService(370): Chore service for: regionserver/4a91c05c96a4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T12:27:16,733 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T12:27:16,733 INFO [regionserver/4a91c05c96a4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T12:27:16,734 INFO [RS:0;4a91c05c96a4:38717 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38717 2024-11-12T12:27:16,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4a91c05c96a4,38717,1731414257691 2024-11-12T12:27:16,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T12:27:16,793 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T12:27:16,793 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4a91c05c96a4,38717,1731414257691] 2024-11-12T12:27:16,803 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4a91c05c96a4,38717,1731414257691 already deleted, retry=false 2024-11-12T12:27:16,803 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4a91c05c96a4,38717,1731414257691 expired; onlineServers=0 2024-11-12T12:27:16,804 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4a91c05c96a4,41939,1731414256942' ***** 2024-11-12T12:27:16,804 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T12:27:16,804 INFO [M:0;4a91c05c96a4:41939 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T12:27:16,804 INFO [M:0;4a91c05c96a4:41939 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T12:27:16,804 DEBUG [M:0;4a91c05c96a4:41939 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T12:27:16,804 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T12:27:16,804 DEBUG [M:0;4a91c05c96a4:41939 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T12:27:16,804 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster-HFileCleaner.large.0-1731414260496 {}] cleaner.HFileCleaner(306): Exit Thread[master/4a91c05c96a4:0:becomeActiveMaster-HFileCleaner.large.0-1731414260496,5,FailOnTimeoutGroup] 2024-11-12T12:27:16,804 DEBUG [master/4a91c05c96a4:0:becomeActiveMaster-HFileCleaner.small.0-1731414260497 {}] cleaner.HFileCleaner(306): Exit Thread[master/4a91c05c96a4:0:becomeActiveMaster-HFileCleaner.small.0-1731414260497,5,FailOnTimeoutGroup] 2024-11-12T12:27:16,804 INFO [M:0;4a91c05c96a4:41939 {}] hbase.ChoreService(370): Chore service for: master/4a91c05c96a4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T12:27:16,804 INFO [M:0;4a91c05c96a4:41939 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T12:27:16,804 DEBUG [M:0;4a91c05c96a4:41939 {}] master.HMaster(1795): Stopping service threads 2024-11-12T12:27:16,804 INFO [M:0;4a91c05c96a4:41939 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T12:27:16,804 INFO [M:0;4a91c05c96a4:41939 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T12:27:16,805 ERROR [M:0;4a91c05c96a4:41939 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:40685 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40685,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-12T12:27:16,805 INFO [M:0;4a91c05c96a4:41939 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T12:27:16,805 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T12:27:16,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T12:27:16,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T12:27:16,814 DEBUG [M:0;4a91c05c96a4:41939 {}] zookeeper.ZKUtil(347): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T12:27:16,814 WARN [M:0;4a91c05c96a4:41939 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T12:27:16,814 INFO [M:0;4a91c05c96a4:41939 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/.lastflushedseqids 2024-11-12T12:27:16,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742429_1605 (size=72) 2024-11-12T12:27:16,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T12:27:16,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38717-0x1012f00e5900001, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T12:27:16,904 INFO [RS:0;4a91c05c96a4:38717 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T12:27:16,904 INFO [RS:0;4a91c05c96a4:38717 {}] regionserver.HRegionServer(1031): Exiting; stopping=4a91c05c96a4,38717,1731414257691; zookeeper connection closed. 2024-11-12T12:27:16,905 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75b59fc8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75b59fc8 2024-11-12T12:27:16,905 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T12:27:17,224 INFO [M:0;4a91c05c96a4:41939 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T12:27:17,224 INFO [M:0;4a91c05c96a4:41939 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T12:27:17,224 DEBUG [M:0;4a91c05c96a4:41939 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T12:27:17,224 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T12:27:17,224 DEBUG [M:0;4a91c05c96a4:41939 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T12:27:17,224 DEBUG [M:0;4a91c05c96a4:41939 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T12:27:17,224 DEBUG [M:0;4a91c05c96a4:41939 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T12:27:17,224 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=741.97 KB heapSize=910.94 KB 2024-11-12T12:27:17,237 DEBUG [M:0;4a91c05c96a4:41939 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/784228edda384de5b96b11ffa997a2ec is 82, key is hbase:meta,,1/info:regioninfo/1731414261686/Put/seqid=0 2024-11-12T12:27:17,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742430_1606 (size=5672) 2024-11-12T12:27:17,286 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T12:27:17,287 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T12:27:17,287 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T12:27:17,641 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2083 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/784228edda384de5b96b11ffa997a2ec 2024-11-12T12:27:17,659 DEBUG [M:0;4a91c05c96a4:41939 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/55bc4a2202ae4c22ada427e343d5ab7f is 2275, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1731414291659/Put/seqid=0 2024-11-12T12:27:17,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742431_1607 (size=43430) 2024-11-12T12:27:18,097 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=741.41 KB at sequenceid=2083 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/55bc4a2202ae4c22ada427e343d5ab7f 2024-11-12T12:27:18,101 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 55bc4a2202ae4c22ada427e343d5ab7f 2024-11-12T12:27:18,117 DEBUG [M:0;4a91c05c96a4:41939 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3279f42d7e8641c9877e7296221f6abd is 69, key is 4a91c05c96a4,38717,1731414257691/rs:state/1731414260631/Put/seqid=0 2024-11-12T12:27:18,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073742432_1608 (size=5156) 2024-11-12T12:27:18,520 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2083 (bloomFilter=true), to=hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3279f42d7e8641c9877e7296221f6abd 2024-11-12T12:27:18,523 DEBUG [M:0;4a91c05c96a4:41939 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/784228edda384de5b96b11ffa997a2ec as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/784228edda384de5b96b11ffa997a2ec 2024-11-12T12:27:18,526 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/784228edda384de5b96b11ffa997a2ec, entries=8, sequenceid=2083, filesize=5.5 K 2024-11-12T12:27:18,526 DEBUG [M:0;4a91c05c96a4:41939 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/55bc4a2202ae4c22ada427e343d5ab7f as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/55bc4a2202ae4c22ada427e343d5ab7f 2024-11-12T12:27:18,528 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 55bc4a2202ae4c22ada427e343d5ab7f 2024-11-12T12:27:18,529 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/55bc4a2202ae4c22ada427e343d5ab7f, entries=166, sequenceid=2083, filesize=42.4 K 2024-11-12T12:27:18,529 DEBUG [M:0;4a91c05c96a4:41939 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3279f42d7e8641c9877e7296221f6abd as hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3279f42d7e8641c9877e7296221f6abd 2024-11-12T12:27:18,531 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40685/user/jenkins/test-data/68cf29d7-b372-b9b2-6f6d-2a248aa4360a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3279f42d7e8641c9877e7296221f6abd, entries=1, sequenceid=2083, filesize=5.0 K 2024-11-12T12:27:18,532 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.HRegion(3140): Finished flush of dataSize ~741.97 KB/759777, heapSize ~910.64 KB/932496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2083, compaction requested=false 2024-11-12T12:27:18,533 INFO [M:0;4a91c05c96a4:41939 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T12:27:18,533 DEBUG [M:0;4a91c05c96a4:41939 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731414437224Disabling compacts and flushes for region at 1731414437224Disabling writes for close at 1731414437224Obtaining lock to block concurrent updates at 1731414437224Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731414437224Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=759777, getHeapSize=932736, getOffHeapSize=0, getCellsCount=2274 at 1731414437224Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731414437225 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731414437225Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731414437237 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731414437237Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731414437644 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731414437658 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731414437658Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731414438101 (+443 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731414438116 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731414438116Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@665cd37e: reopening flushed file at 1731414438523 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d0ffd5c: reopening flushed file at 1731414438526 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aff9049: reopening flushed file at 1731414438529 (+3 ms)Finished flush of dataSize ~741.97 KB/759777, heapSize ~910.64 KB/932496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2083, compaction requested=false at 1731414438532 (+3 ms)Writing region close event to WAL at 1731414438533 (+1 ms)Closed at 1731414438533 2024-11-12T12:27:18,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33211 is added to blk_1073741830_1006 (size=896355) 2024-11-12T12:27:18,535 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T12:27:18,536 INFO [M:0;4a91c05c96a4:41939 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T12:27:18,536 INFO [M:0;4a91c05c96a4:41939 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41939 2024-11-12T12:27:18,536 INFO [M:0;4a91c05c96a4:41939 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T12:27:18,651 INFO [M:0;4a91c05c96a4:41939 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T12:27:18,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T12:27:18,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41939-0x1012f00e5900000, quorum=127.0.0.1:59170, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T12:27:18,688 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T12:27:18,690 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T12:27:18,690 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T12:27:18,690 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T12:27:18,690 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/hadoop.log.dir/,STOPPED} 2024-11-12T12:27:18,693 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T12:27:18,693 WARN [BP-1782012263-172.17.0.3-1731414252404 heartbeating to localhost/127.0.0.1:40685 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T12:27:18,693 WARN [BP-1782012263-172.17.0.3-1731414252404 heartbeating to localhost/127.0.0.1:40685 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1782012263-172.17.0.3-1731414252404 (Datanode Uuid 927edd63-cfe6-496c-9696-870b8024df37) service to localhost/127.0.0.1:40685 2024-11-12T12:27:18,693 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T12:27:18,695 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269/data/data1/current/BP-1782012263-172.17.0.3-1731414252404 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T12:27:18,695 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/cluster_d9e1bc67-5d95-daec-c802-ed3e92d5f269/data/data2/current/BP-1782012263-172.17.0.3-1731414252404 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T12:27:18,696 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T12:27:18,704 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T12:27:18,704 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T12:27:18,704 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T12:27:18,705 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T12:27:18,705 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/59714935-b4c7-3ddf-0452-550844af51f3/hadoop.log.dir/,STOPPED} 2024-11-12T12:27:18,719 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T12:27:18,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down